From 7375d03990b686922e06a4ebeccbb175ba7dc5ea Mon Sep 17 00:00:00 2001 From: Nick Miyake Date: Mon, 9 Sep 2024 13:48:03 -0700 Subject: [PATCH] Update logic to use mholt/archiver/v3 --- go.mod | 18 +- go.sum | 48 +- ir-gen-cli-bundler/conjureircli/run.go | 16 +- .../github.com/andybalholm/brotli/README.md | 9 + .../andybalholm/brotli/backward_references.go | 36 +- .../brotli/backward_references_hq.go | 37 +- .../andybalholm/brotli/bitwriter.go | 56 + .../andybalholm/brotli/block_splitter.go | 37 +- .../brotli/block_splitter_command.go | 3 +- .../andybalholm/brotli/brotli_bit_stream.go | 488 ++++- .../andybalholm/brotli/cluster_command.go | 162 -- .../github.com/andybalholm/brotli/command.go | 24 +- .../andybalholm/brotli/compress_fragment.go | 12 +- .../brotli/compress_fragment_two_pass.go | 64 +- .../github.com/andybalholm/brotli/decode.go | 77 +- .../github.com/andybalholm/brotli/encode.go | 807 ++------ .../github.com/andybalholm/brotli/encoder.go | 168 ++ .../andybalholm/brotli/entropy_encode.go | 13 +- .../brotli/entropy_encode_static.go | 5 + .../github.com/andybalholm/brotli/fast_log.go | 16 +- .../andybalholm/brotli/find_match_length.go | 29 + vendor/github.com/andybalholm/brotli/hash.go | 2 - .../brotli/hash_forgetful_chain.go | 3 +- .../andybalholm/brotli/hash_rolling.go | 1 - .../andybalholm/brotli/histogram.go | 7 +- vendor/github.com/andybalholm/brotli/http.go | 184 ++ .../andybalholm/brotli/matchfinder/emitter.go | 45 + .../andybalholm/brotli/matchfinder/m0.go | 169 ++ .../andybalholm/brotli/matchfinder/m4.go | 297 +++ .../brotli/matchfinder/matchfinder.go | 103 + .../brotli/matchfinder/textencoder.go | 53 + .../github.com/andybalholm/brotli/memory.go | 28 +- .../andybalholm/brotli/metablock.go | 141 +- .../andybalholm/brotli/metablock_command.go | 7 +- .../andybalholm/brotli/metablock_distance.go | 7 +- .../andybalholm/brotli/metablock_literal.go | 7 +- .../github.com/andybalholm/brotli/reader.go | 12 +- .../andybalholm/brotli/ringbuffer.go | 12 +- vendor/github.com/andybalholm/brotli/state.go | 1 - .../andybalholm/brotli/static_dict.go | 12 +- .../andybalholm/brotli/utf8_util.go | 3 +- .../andybalholm/brotli/write_bits.go | 26 +- .../github.com/andybalholm/brotli/writer.go | 89 +- vendor/github.com/golang/gddo/LICENSE | 27 - .../github.com/golang/gddo/httputil/buster.go | 95 - .../golang/gddo/httputil/header/header.go | 298 --- .../golang/gddo/httputil/httputil.go | 25 - .../golang/gddo/httputil/negotiate.go | 79 - .../golang/gddo/httputil/respbuf.go | 58 - .../github.com/golang/gddo/httputil/static.go | 265 --- .../golang/gddo/httputil/transport.go | 87 - .../klauspost/compress/gzip/gunzip.go | 374 ++++ .../klauspost/compress/gzip/gzip.go | 269 +++ .../klauspost/compress/zip/reader.go | 905 ++++++++ .../klauspost/compress/zip/register.go | 148 ++ .../klauspost/compress/zip/struct.go | 392 ++++ .../klauspost/compress/zip/writer.go | 640 ++++++ vendor/github.com/klauspost/pgzip/.travis.yml | 17 +- vendor/github.com/klauspost/pgzip/LICENSE | 3 +- vendor/github.com/klauspost/pgzip/README.md | 17 +- vendor/github.com/klauspost/pgzip/gunzip.go | 11 + vendor/github.com/klauspost/pgzip/gzip.go | 80 +- vendor/github.com/mholt/archiver/.gitignore | 5 - vendor/github.com/mholt/archiver/.travis.yml | 21 - vendor/github.com/mholt/archiver/LICENSE | 21 - vendor/github.com/mholt/archiver/README.md | 110 - vendor/github.com/mholt/archiver/appveyor.yml | 32 - vendor/github.com/mholt/archiver/archiver.go | 119 -- vendor/github.com/mholt/archiver/build.bash | 21 - vendor/github.com/mholt/archiver/rar.go | 120 -- vendor/github.com/mholt/archiver/tar.go | 244 --- vendor/github.com/mholt/archiver/tarbz2.go | 106 - vendor/github.com/mholt/archiver/targz.go | 98 - vendor/github.com/mholt/archiver/tarlz4.go | 92 - vendor/github.com/mholt/archiver/tarsz.go | 92 - vendor/github.com/mholt/archiver/tarxz.go | 105 - .../github.com/mholt/archiver/v3/.gitignore | 7 +- .../mholt/archiver/v3/.goreleaser.yml | 41 + .../github.com/mholt/archiver/v3/.prettierrc | 4 + vendor/github.com/mholt/archiver/v3/README.md | 101 +- .../github.com/mholt/archiver/v3/SECURITY.md | 15 + .../github.com/mholt/archiver/v3/archiver.go | 8 + .../mholt/archiver/v3/azure-pipelines.yml | 86 - vendor/github.com/mholt/archiver/v3/error.go | 27 + vendor/github.com/mholt/archiver/v3/gz.go | 4 +- vendor/github.com/mholt/archiver/v3/lz4.go | 11 +- vendor/github.com/mholt/archiver/v3/rar.go | 45 +- vendor/github.com/mholt/archiver/v3/sz.go | 2 +- vendor/github.com/mholt/archiver/v3/tar.go | 77 +- vendor/github.com/mholt/archiver/v3/targz.go | 4 +- vendor/github.com/mholt/archiver/v3/tarlz4.go | 11 +- vendor/github.com/mholt/archiver/v3/tarsz.go | 2 +- vendor/github.com/mholt/archiver/v3/zip.go | 142 +- vendor/github.com/mholt/archiver/v3/zstd.go | 6 +- vendor/github.com/mholt/archiver/zip.go | 238 --- .../github.com/nwaples/rardecode/archive.go | 15 +- .../github.com/nwaples/rardecode/huffman.go | 2 +- vendor/github.com/nwaples/rardecode/reader.go | 7 + .../builtintasks/packages/packages.go | 2 +- .../defaulttasks/defaulttasks.go | 170 +- vendor/github.com/palantir/pkg/pkgpath/godelw | 10 +- .../palantir/pkg/pkgpath/packages.go | 128 +- vendor/github.com/pierrec/lz4/.travis.yml | 18 - vendor/github.com/pierrec/lz4/README.md | 24 - vendor/github.com/pierrec/lz4/block.go | 397 ---- vendor/github.com/pierrec/lz4/debug.go | 23 - vendor/github.com/pierrec/lz4/debug_stub.go | 7 - .../pierrec/lz4/internal/xxh32/xxh32zero.go | 222 -- vendor/github.com/pierrec/lz4/lz4.go | 68 - vendor/github.com/pierrec/lz4/lz4_go1.10.go | 29 - .../github.com/pierrec/lz4/lz4_notgo1.10.go | 29 - vendor/github.com/pierrec/lz4/reader.go | 295 --- .../pierrec/lz4/{ => v4}/.gitignore | 5 +- .../github.com/pierrec/lz4/{ => v4}/LICENSE | 0 vendor/github.com/pierrec/lz4/v4/README.md | 90 + .../pierrec/lz4/v4/internal/lz4block/block.go | 469 +++++ .../lz4/v4/internal/lz4block/blocks.go | 88 + .../lz4/v4/internal/lz4block/decode_amd64.s | 369 ++++ .../lz4/v4/internal/lz4block/decode_arm.s | 197 ++ .../lz4/v4/internal/lz4block/decode_asm.go | 9 + .../lz4/v4/internal/lz4block/decode_other.go | 108 + .../lz4/v4/internal/lz4errors/errors.go | 19 + .../lz4/v4/internal/lz4stream/block.go | 332 +++ .../lz4/v4/internal/lz4stream/frame.go | 200 ++ .../lz4/v4/internal/lz4stream/frame_gen.go | 103 + .../lz4/v4/internal/xxh32/xxh32zero.go | 212 ++ .../lz4/v4/internal/xxh32/xxh32zero_arm.go | 11 + .../lz4/v4/internal/xxh32/xxh32zero_arm.s | 259 +++ .../lz4/v4/internal/xxh32/xxh32zero_other.go | 10 + vendor/github.com/pierrec/lz4/v4/lz4.go | 147 ++ vendor/github.com/pierrec/lz4/v4/options.go | 213 ++ .../github.com/pierrec/lz4/v4/options_gen.go | 92 + vendor/github.com/pierrec/lz4/v4/reader.go | 243 +++ vendor/github.com/pierrec/lz4/v4/state.go | 75 + vendor/github.com/pierrec/lz4/v4/state_gen.go | 28 + vendor/github.com/pierrec/lz4/v4/writer.go | 233 +++ vendor/github.com/pierrec/lz4/writer.go | 267 --- vendor/golang.org/x/mod/modfile/print.go | 184 ++ vendor/golang.org/x/mod/modfile/read.go | 959 +++++++++ vendor/golang.org/x/mod/modfile/rule.go | 1836 +++++++++++++++++ vendor/golang.org/x/mod/modfile/work.go | 335 +++ vendor/modules.txt | 44 +- 142 files changed, 12340 insertions(+), 5384 deletions(-) create mode 100644 vendor/github.com/andybalholm/brotli/bitwriter.go create mode 100644 vendor/github.com/andybalholm/brotli/encoder.go create mode 100644 vendor/github.com/andybalholm/brotli/http.go create mode 100644 vendor/github.com/andybalholm/brotli/matchfinder/emitter.go create mode 100644 vendor/github.com/andybalholm/brotli/matchfinder/m0.go create mode 100644 vendor/github.com/andybalholm/brotli/matchfinder/m4.go create mode 100644 vendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go create mode 100644 vendor/github.com/andybalholm/brotli/matchfinder/textencoder.go delete mode 100644 vendor/github.com/golang/gddo/LICENSE delete mode 100644 vendor/github.com/golang/gddo/httputil/buster.go delete mode 100644 vendor/github.com/golang/gddo/httputil/header/header.go delete mode 100644 vendor/github.com/golang/gddo/httputil/httputil.go delete mode 100644 vendor/github.com/golang/gddo/httputil/negotiate.go delete mode 100644 vendor/github.com/golang/gddo/httputil/respbuf.go delete mode 100644 vendor/github.com/golang/gddo/httputil/static.go delete mode 100644 vendor/github.com/golang/gddo/httputil/transport.go create mode 100644 vendor/github.com/klauspost/compress/gzip/gunzip.go create mode 100644 vendor/github.com/klauspost/compress/gzip/gzip.go create mode 100644 vendor/github.com/klauspost/compress/zip/reader.go create mode 100644 vendor/github.com/klauspost/compress/zip/register.go create mode 100644 vendor/github.com/klauspost/compress/zip/struct.go create mode 100644 vendor/github.com/klauspost/compress/zip/writer.go delete mode 100644 vendor/github.com/mholt/archiver/.gitignore delete mode 100644 vendor/github.com/mholt/archiver/.travis.yml delete mode 100644 vendor/github.com/mholt/archiver/LICENSE delete mode 100644 vendor/github.com/mholt/archiver/README.md delete mode 100644 vendor/github.com/mholt/archiver/appveyor.yml delete mode 100644 vendor/github.com/mholt/archiver/archiver.go delete mode 100644 vendor/github.com/mholt/archiver/build.bash delete mode 100644 vendor/github.com/mholt/archiver/rar.go delete mode 100644 vendor/github.com/mholt/archiver/tar.go delete mode 100644 vendor/github.com/mholt/archiver/tarbz2.go delete mode 100644 vendor/github.com/mholt/archiver/targz.go delete mode 100644 vendor/github.com/mholt/archiver/tarlz4.go delete mode 100644 vendor/github.com/mholt/archiver/tarsz.go delete mode 100644 vendor/github.com/mholt/archiver/tarxz.go create mode 100644 vendor/github.com/mholt/archiver/v3/.goreleaser.yml create mode 100644 vendor/github.com/mholt/archiver/v3/.prettierrc create mode 100644 vendor/github.com/mholt/archiver/v3/SECURITY.md delete mode 100644 vendor/github.com/mholt/archiver/v3/azure-pipelines.yml create mode 100644 vendor/github.com/mholt/archiver/v3/error.go delete mode 100644 vendor/github.com/mholt/archiver/zip.go delete mode 100644 vendor/github.com/pierrec/lz4/.travis.yml delete mode 100644 vendor/github.com/pierrec/lz4/README.md delete mode 100644 vendor/github.com/pierrec/lz4/block.go delete mode 100644 vendor/github.com/pierrec/lz4/debug.go delete mode 100644 vendor/github.com/pierrec/lz4/debug_stub.go delete mode 100644 vendor/github.com/pierrec/lz4/internal/xxh32/xxh32zero.go delete mode 100644 vendor/github.com/pierrec/lz4/lz4.go delete mode 100644 vendor/github.com/pierrec/lz4/lz4_go1.10.go delete mode 100644 vendor/github.com/pierrec/lz4/lz4_notgo1.10.go delete mode 100644 vendor/github.com/pierrec/lz4/reader.go rename vendor/github.com/pierrec/lz4/{ => v4}/.gitignore (94%) rename vendor/github.com/pierrec/lz4/{ => v4}/LICENSE (100%) create mode 100644 vendor/github.com/pierrec/lz4/v4/README.md create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_amd64.s create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_asm.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4errors/errors.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame_gen.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.s create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_other.go create mode 100644 vendor/github.com/pierrec/lz4/v4/lz4.go create mode 100644 vendor/github.com/pierrec/lz4/v4/options.go create mode 100644 vendor/github.com/pierrec/lz4/v4/options_gen.go create mode 100644 vendor/github.com/pierrec/lz4/v4/reader.go create mode 100644 vendor/github.com/pierrec/lz4/v4/state.go create mode 100644 vendor/github.com/pierrec/lz4/v4/state_gen.go create mode 100644 vendor/github.com/pierrec/lz4/v4/writer.go delete mode 100644 vendor/github.com/pierrec/lz4/writer.go create mode 100644 vendor/golang.org/x/mod/modfile/print.go create mode 100644 vendor/golang.org/x/mod/modfile/read.go create mode 100644 vendor/golang.org/x/mod/modfile/rule.go create mode 100644 vendor/golang.org/x/mod/modfile/work.go diff --git a/go.mod b/go.mod index e4eee64c..6e745f64 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module github.com/palantir/godel-conjure-plugin/v6 go 1.22 require ( - github.com/mholt/archiver v2.1.0+incompatible + github.com/mholt/archiver/v3 v3.5.1 github.com/nmiyake/pkg/dirs v1.1.0 github.com/palantir/conjure-go/v6 v6.60.0 github.com/palantir/distgo v1.75.0 - github.com/palantir/godel/v2 v2.114.0 + github.com/palantir/godel/v2 v2.118.0 github.com/palantir/pkg/cobracli v1.2.0 github.com/palantir/pkg/safehttp v1.1.0 github.com/palantir/pkg/safejson v1.1.0 @@ -19,34 +19,32 @@ require ( require ( github.com/VividCortex/ewma v1.1.1 // indirect - github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect github.com/cheggaaa/pb/v3 v3.0.2 // indirect github.com/dave/jennifer v1.4.2-0.20211112003305-45cc0b7eb71a // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dsnet/compress v0.0.1 // indirect + github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/fatih/color v1.13.0 // indirect - github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/compress v1.16.5 // indirect - github.com/klauspost/pgzip v1.2.1 // indirect + github.com/klauspost/pgzip v1.2.5 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.4 // indirect - github.com/mholt/archiver/v3 v3.3.0 // indirect github.com/nmiyake/pkg/errorstringer v1.1.0 // indirect - github.com/nwaples/rardecode v1.0.0 // indirect + github.com/nwaples/rardecode v1.1.0 // indirect github.com/palantir/distgo/pkg/git v1.0.0 // indirect github.com/palantir/go-ptimports/v2 v2.10.0 // indirect github.com/palantir/pkg v1.1.0 // indirect github.com/palantir/pkg/matcher v1.2.0 // indirect - github.com/palantir/pkg/pkgpath v1.2.0 // indirect + github.com/palantir/pkg/pkgpath v1.3.0 // indirect github.com/palantir/pkg/safeyaml v1.1.0 // indirect github.com/palantir/pkg/specdir v1.2.0 // indirect github.com/palantir/pkg/transform v1.1.0 // indirect github.com/palantir/witchcraft-go-error v1.39.0 // indirect github.com/palantir/witchcraft-go-params v1.36.0 // indirect - github.com/pierrec/lz4 v2.0.5+incompatible // indirect + github.com/pierrec/lz4/v4 v4.1.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index 86c284c9..2672218f 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,9 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6 h1:bZ28Hqta7TFAK3Q08CMvv8y3/8ATaEqv2nGoc6yff6c= -github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6/go.mod h1:+lx6/Aqd1kLJ1GQfkvOnaZ1WGmLpMpbprPuIOOZX30U= +github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -50,8 +51,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= @@ -65,8 +66,6 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721 h1:KRMr9A3qfbVM7iV/WcLY/rL5LICqwMHLhwRXKu99fXw= -github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -75,15 +74,14 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -126,12 +124,12 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM= -github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -157,10 +155,8 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mholt/archiver v2.1.0+incompatible h1:1ivm7KAHPtPere1YDOdrY6xGdbMNGRWThZbYh5lWZT0= -github.com/mholt/archiver v2.1.0+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= -github.com/mholt/archiver/v3 v3.3.0 h1:vWjhY8SQp5yzM9P6OJ/eZEkmi3UAbRrxCq48MxjAzig= -github.com/mholt/archiver/v3 v3.3.0/go.mod h1:YnQtqsp+94Rwd0D/rk5cnLrxusUBUXg+08Ebtr1Mqao= +github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= +github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -181,8 +177,8 @@ github.com/nmiyake/pkg/errorstringer v1.1.0 h1:A50S8deDIe+otJNB/BZKpDb5a3E4IUTOg github.com/nmiyake/pkg/errorstringer v1.1.0/go.mod h1:3c/Luj1o1XtUmdqL6QRKnBnkfBXNU5zU8a2vDxi9AF4= github.com/nmiyake/pkg/gofiles v1.2.0 h1:L0LWMfnHyMxiLP2Cuqno163/4zK73FscEsNd7sHo2M4= github.com/nmiyake/pkg/gofiles v1.2.0/go.mod h1:aPXiVvXPwxNanRNuFIRVPYn6eTC4qxjDZni6VNNlzMY= -github.com/nwaples/rardecode v1.0.0 h1:r7vGuS5akxOnR4JQSkko62RJ1ReCMXxQRPtxsiFMBOs= -github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/palantir/conjure-go/v6 v6.60.0 h1:53s2xeqaggHLzq6KUebZWDuAqbNvA9PdOlo8MwTmJfM= github.com/palantir/conjure-go/v6 v6.60.0/go.mod h1:OgL0bUTlpPWQ8s6wbg0zcJzqs/6419xK/yEADhreEbo= @@ -192,8 +188,8 @@ github.com/palantir/distgo/pkg/git v1.0.0 h1:bryRJ9ZdJapz5tGzd/snv6U66tLxyb1b4Pb github.com/palantir/distgo/pkg/git v1.0.0/go.mod h1:eXrr3SOmf/sWTYmtiubYVxSaVegTlMGgRgBUFMFFedw= github.com/palantir/go-ptimports/v2 v2.10.0 h1:foAjyLulrcRc89Vto7EwniJJDiQT8oiG8CsLAH9h8HE= github.com/palantir/go-ptimports/v2 v2.10.0/go.mod h1:BhZDs/eL/RvgShGCPEG+vKxZJug+geeGEppwLjedxNI= -github.com/palantir/godel/v2 v2.114.0 h1:pBoupWHWOmuut4Zz/ivZLdHl0+Xv2fvgpFipVq/sKrs= -github.com/palantir/godel/v2 v2.114.0/go.mod h1:ofxevX9KEQ6UdpeX80PPR/0uJ2Y107eJCLFNtMeQyCw= +github.com/palantir/godel/v2 v2.118.0 h1:gtdImXmldlGygri2+3wNbBH8Wo4LExWRd3FYBAlMn+4= +github.com/palantir/godel/v2 v2.118.0/go.mod h1:TVDmWkNg0Iv1C1jDG9QlRC596OJcFboL9/+K5OK9TSk= github.com/palantir/pkg v1.0.1/go.mod h1:Eo6Jl0UXfT+65sLXJOcU9duu0WPvKsWFXCb0dE5VWZs= github.com/palantir/pkg v1.1.0 h1:0EhrSUP8oeeh3MUvk7V/UU7WmsN1UiJNTvNj0sN9Cpo= github.com/palantir/pkg v1.1.0/go.mod h1:KC9srP/9ssWRxBxFCIqhUGC4Jt7OJkWRz0Iqehup1/c= @@ -205,8 +201,8 @@ github.com/palantir/pkg/gittest v1.2.0 h1:RPINGtLh+Mlt+o1M/nRQlTPrt4CP5tJOSMgeB5 github.com/palantir/pkg/gittest v1.2.0/go.mod h1:z2eq/rEzUDmNR8rvJCb+/XZ7w9/nxRCAKvYjMYW/w/w= github.com/palantir/pkg/matcher v1.2.0 h1:h4IeYPSQGWIdi1Qh7QSzWATv0+2coTaaCiozYtPWBks= github.com/palantir/pkg/matcher v1.2.0/go.mod h1:JUH9L+Cmjv2U87y+1Ov5KKLmMbgHtESCTrPq5MyWeVM= -github.com/palantir/pkg/pkgpath v1.2.0 h1:6JDFKFZZsiVBiek47OB7363oIAIjQNFkTsPoEtJpWzY= -github.com/palantir/pkg/pkgpath v1.2.0/go.mod h1:VaK3q7OPEFs+YqIAAJBi3oE+dw622WmucIRpU4qa14g= +github.com/palantir/pkg/pkgpath v1.3.0 h1:S2kbdqSEMSHLgaEdr8LQuCju+AfzSvwwSUM/Q4QUNUI= +github.com/palantir/pkg/pkgpath v1.3.0/go.mod h1:22X+LjC1DKT6Zc9KYTPXOc0AP95gzPIC3me45vz78ac= github.com/palantir/pkg/safehttp v1.1.0 h1:S4r6xhgc01azkP6L289QqJlawOe+EE5o/Jx3XNZHzus= github.com/palantir/pkg/safehttp v1.1.0/go.mod h1:AZs3pGxwx7Iyxg+csUOxUfARkJ/JJJ08yywBFl1RuDs= github.com/palantir/pkg/safejson v1.1.0 h1:myVdz3dGPjac1o3aYDuODk7BaYdsxHcYLfVNXMRP+MU= @@ -223,8 +219,8 @@ github.com/palantir/witchcraft-go-params v1.36.0 h1:/YF0RJEqEt/4/t9gP4vCU74P9wUi github.com/palantir/witchcraft-go-params v1.36.0/go.mod h1:R+/PmtwK5BfCIrA6JFlUGhJfhTQOHGjQaLAaUvqfPLo= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= +github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -280,7 +276,8 @@ github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae h1:vgGSvdW5Lqg+I1 github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae/go.mod h1:quDq6Se6jlGwiIKia/itDZxqC5rj6/8OdFyMMAwTxCs= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU= @@ -404,6 +401,7 @@ golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= diff --git a/ir-gen-cli-bundler/conjureircli/run.go b/ir-gen-cli-bundler/conjureircli/run.go index a579227e..b3d30c5c 100644 --- a/ir-gen-cli-bundler/conjureircli/run.go +++ b/ir-gen-cli-bundler/conjureircli/run.go @@ -15,16 +15,16 @@ package conjureircli import ( - "bytes" _ "embed" // required for go:embed directive "fmt" "io/ioutil" "os" "os/exec" "path" + "path/filepath" "runtime" - "github.com/mholt/archiver" + "github.com/mholt/archiver/v3" "github.com/palantir/godel-conjure-plugin/v6/ir-gen-cli-bundler/conjureircli/internal" "github.com/palantir/pkg/safejson" "github.com/pkg/errors" @@ -184,7 +184,17 @@ func ensureCLIExists(cliPath string) error { } // expand asset into destination - if err := archiver.TarGz.Read(bytes.NewReader(conjureCliTGZ), cliUnpackDir); err != nil { + tmpDir, err := os.MkdirTemp("", "conjure-cli-*") + if err != nil { + return errors.Wrap(err, "failed to create temporary directory for CLI TGZ") + } + tmpTGZPath := filepath.Join(tmpDir, "conjure-cli.tgz") + if err := os.WriteFile(tmpTGZPath, conjureCliTGZ, 0644); err != nil { + return errors.Wrap(err, "failed to write Conjure CLI TGZ") + } + + tarGZ := &archiver.TarGz{} + if err := tarGZ.Unarchive(tmpTGZPath, cliUnpackDir); err != nil { return errors.WithStack(err) } diff --git a/vendor/github.com/andybalholm/brotli/README.md b/vendor/github.com/andybalholm/brotli/README.md index 02d115e5..00625211 100644 --- a/vendor/github.com/andybalholm/brotli/README.md +++ b/vendor/github.com/andybalholm/brotli/README.md @@ -2,4 +2,13 @@ This package is a brotli compressor and decompressor implemented in Go. It was translated from the reference implementation (https://github.com/google/brotli) with the `c2go` tool at https://github.com/andybalholm/c2go. +I have been working on new compression algorithms (not translated from C) +in the matchfinder package. +You can use them with the NewWriterV2 function. +Currently they give better results than the old implementation +(at least for compressing my test file, Newton’s *Opticks*) +on levels 2 to 6. + I am using it in production with https://github.com/andybalholm/redwood. + +API documentation is found at https://pkg.go.dev/github.com/andybalholm/brotli?tab=doc. diff --git a/vendor/github.com/andybalholm/brotli/backward_references.go b/vendor/github.com/andybalholm/brotli/backward_references.go index 1642e471..008c054d 100644 --- a/vendor/github.com/andybalholm/brotli/backward_references.go +++ b/vendor/github.com/andybalholm/brotli/backward_references.go @@ -1,5 +1,9 @@ package brotli +import ( + "sync" +) + /* Copyright 2013 Google Inc. All Rights Reserved. Distributed under MIT license. @@ -31,13 +35,10 @@ func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uin return distance + numDistanceShortCodes - 1 } -/* "commands" points to the next output command to write to, "*num_commands" is - initially the total amount of commands output by previous - CreateBackwardReferences calls, and must be incremented by the amount written - by this call. */ -func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { +var hasherSearchResultPool sync.Pool + +func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) { var max_backward_limit uint = maxBackwardLimit(params.lgwin) - var orig_commands []command = commands var insert_length uint = *last_insert_len var pos_end uint = position + num_bytes var store_end uint @@ -57,8 +58,14 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, /* Minimum score to accept a backward reference. */ hasher.PrepareDistanceCache(dist_cache) - var sr2 hasherSearchResult - var sr hasherSearchResult + sr2, _ := hasherSearchResultPool.Get().(*hasherSearchResult) + if sr2 == nil { + sr2 = &hasherSearchResult{} + } + sr, _ := hasherSearchResultPool.Get().(*hasherSearchResult) + if sr == nil { + sr = &hasherSearchResult{} + } for position+hasher.HashTypeLength() < pos_end { var max_length uint = pos_end - position @@ -67,7 +74,7 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, sr.len_code_delta = 0 sr.distance = 0 sr.score = kMinScore - hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position, max_length, max_distance, gap, params.dist.max_distance, &sr) + hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position, max_length, max_distance, gap, params.dist.max_distance, sr) if sr.score > kMinScore { /* Found a match. Let's look for something even better ahead. */ var delayed_backward_references_in_row int = 0 @@ -83,14 +90,14 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, sr2.distance = 0 sr2.score = kMinScore max_distance = brotli_min_size_t(position+1, max_backward_limit) - hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position+1, max_length, max_distance, gap, params.dist.max_distance, &sr2) + hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position+1, max_length, max_distance, gap, params.dist.max_distance, sr2) if sr2.score >= sr.score+cost_diff_lazy { /* Ok, let's just write one byte for now and start a match from the next byte. */ position++ insert_length++ - sr = sr2 + *sr = *sr2 delayed_backward_references_in_row++ if delayed_backward_references_in_row < 4 && position+hasher.HashTypeLength() < pos_end { continue @@ -114,8 +121,7 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, hasher.PrepareDistanceCache(dist_cache) } - initCommand(&commands[0], ¶ms.dist, insert_length, sr.len, sr.len_code_delta, distance_code) - commands = commands[1:] + *commands = append(*commands, makeCommand(¶ms.dist, insert_length, sr.len, sr.len_code_delta, distance_code)) } *num_literals += insert_length @@ -173,5 +179,7 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, insert_length += pos_end - position *last_insert_len = insert_length - *num_commands += uint(-cap(commands) + cap(orig_commands)) + + hasherSearchResultPool.Put(sr) + hasherSearchResultPool.Put(sr2) } diff --git a/vendor/github.com/andybalholm/brotli/backward_references_hq.go b/vendor/github.com/andybalholm/brotli/backward_references_hq.go index 5eac7361..21629c1c 100644 --- a/vendor/github.com/andybalholm/brotli/backward_references_hq.go +++ b/vendor/github.com/andybalholm/brotli/backward_references_hq.go @@ -123,14 +123,13 @@ func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, co } } -func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, num_commands uint, last_insert_len uint) { +func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, last_insert_len uint) { var histogram_literal [numLiteralSymbols]uint32 var histogram_cmd [numCommandSymbols]uint32 var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32 var cost_literal [numLiteralSymbols]float32 var pos uint = position - last_insert_len var min_cost_cmd float32 = kInfinity - var i uint var cost_cmd []float32 = self.cost_cmd_[:] var literal_costs []float32 @@ -138,7 +137,7 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu histogram_cmd = [numCommandSymbols]uint32{} histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{} - for i = 0; i < num_commands; i++ { + for i := range commands { var inslength uint = uint(commands[i].insert_len_) var copylength uint = uint(commandCopyLen(&commands[i])) var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF @@ -161,7 +160,7 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd) setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_) - for i = 0; i < numCommandSymbols; i++ { + for i := 0; i < numCommandSymbols; i++ { min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i]) } @@ -169,10 +168,10 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu { literal_costs = self.literal_costs_ var literal_carry float32 = 0.0 - var num_bytes uint = self.num_bytes_ + num_bytes := int(self.num_bytes_) literal_costs[0] = 0.0 - for i = 0; i < num_bytes; i++ { - literal_carry += cost_literal[ringbuffer[(position+i)&ringbuffer_mask]] + for i := 0; i < num_bytes; i++ { + literal_carry += cost_literal[ringbuffer[(position+uint(i))&ringbuffer_mask]] literal_costs[i+1] = literal_costs[i] + literal_carry literal_carry -= literal_costs[i+1] - literal_costs[i] } @@ -502,7 +501,9 @@ func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte, var cost float32 = dist_cost + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode) if cost < nodes[pos+len].u.cost { updateZopfliNode(nodes, pos, start, uint(len), len_code, dist, 0, cost) - result = brotli_max_size_t(result, uint(len)) + if len > result { + result = len + } } } } @@ -530,7 +531,7 @@ func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint { } /* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */ -func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands []command, num_literals *uint) { +func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands *[]command, num_literals *uint) { var max_backward_limit uint = maxBackwardLimit(params.lgwin) var pos uint = 0 var offset uint32 = nodes[0].u.next @@ -552,7 +553,7 @@ func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit) var is_dictionary bool = (distance > max_distance+gap) var dist_code uint = uint(zopfliNodeDistanceCode(next)) - initCommand(&commands[i], ¶ms.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code) + *commands = append(*commands, makeCommand(¶ms.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code)) if !is_dictionary && dist_code > 0 { dist_cache[3] = dist_cache[2] @@ -679,16 +680,16 @@ func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, return computeShortestPathFromNodes(num_bytes, nodes) } -func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { +func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) { var nodes []zopfliNode nodes = make([]zopfliNode, (num_bytes + 1)) initZopfliNodes(nodes, num_bytes+1) - *num_commands += zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes) + zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes) zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) nodes = nil } -func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { +func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) { var max_backward_limit uint = maxBackwardLimit(params.lgwin) var num_matches []uint32 = make([]uint32, num_bytes) var matches_size uint = 4 * num_bytes @@ -703,7 +704,7 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer var orig_num_literals uint var orig_last_insert_len uint var orig_dist_cache [4]int - var orig_num_commands uint + var orig_num_commands int var model zopfliCostModel var nodes []zopfliNode var matches []backwardMatch = make([]backwardMatch, matches_size) @@ -769,7 +770,7 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer orig_num_literals = *num_literals orig_last_insert_len = *last_insert_len copy(orig_dist_cache[:], dist_cache[:4]) - orig_num_commands = *num_commands + orig_num_commands = len(*commands) nodes = make([]zopfliNode, (num_bytes + 1)) initZopfliCostModel(&model, ¶ms.dist, num_bytes) for i = 0; i < 2; i++ { @@ -777,14 +778,14 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer if i == 0 { zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask) } else { - zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, commands, *num_commands-orig_num_commands, orig_last_insert_len) + zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, (*commands)[orig_num_commands:], orig_last_insert_len) } - *num_commands = orig_num_commands + *commands = (*commands)[:orig_num_commands] *num_literals = orig_num_literals *last_insert_len = orig_last_insert_len copy(dist_cache, orig_dist_cache[:4]) - *num_commands += zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes) + zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes) zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) } diff --git a/vendor/github.com/andybalholm/brotli/bitwriter.go b/vendor/github.com/andybalholm/brotli/bitwriter.go new file mode 100644 index 00000000..dfc60360 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/bitwriter.go @@ -0,0 +1,56 @@ +package brotli + +/* Copyright 2010 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Write bits into a byte array. */ + +type bitWriter struct { + dst []byte + + // Data waiting to be written is the low nbits of bits. + bits uint64 + nbits uint +} + +func (w *bitWriter) writeBits(nb uint, b uint64) { + w.bits |= b << w.nbits + w.nbits += nb + if w.nbits >= 32 { + bits := w.bits + w.bits >>= 32 + w.nbits -= 32 + w.dst = append(w.dst, + byte(bits), + byte(bits>>8), + byte(bits>>16), + byte(bits>>24), + ) + } +} + +func (w *bitWriter) writeSingleBit(bit bool) { + if bit { + w.writeBits(1, 1) + } else { + w.writeBits(1, 0) + } +} + +func (w *bitWriter) jumpToByteBoundary() { + dst := w.dst + for w.nbits != 0 { + dst = append(dst, byte(w.bits)) + w.bits >>= 8 + if w.nbits > 8 { // Avoid underflow + w.nbits -= 8 + } else { + w.nbits = 0 + } + } + w.bits = 0 + w.dst = dst +} diff --git a/vendor/github.com/andybalholm/brotli/block_splitter.go b/vendor/github.com/andybalholm/brotli/block_splitter.go index 2ccff45a..978a1314 100644 --- a/vendor/github.com/andybalholm/brotli/block_splitter.go +++ b/vendor/github.com/andybalholm/brotli/block_splitter.go @@ -33,23 +33,21 @@ const ( kMinItersForRefining uint = 100 ) -func countLiterals(cmds []command, num_commands uint) uint { +func countLiterals(cmds []command) uint { var total_length uint = 0 /* Count how many we have. */ - var i uint - for i = 0; i < num_commands; i++ { + for i := range cmds { total_length += uint(cmds[i].insert_len_) } return total_length } -func copyLiteralsToByteArray(cmds []command, num_commands uint, data []byte, offset uint, mask uint, literals []byte) { +func copyLiteralsToByteArray(cmds []command, data []byte, offset uint, mask uint, literals []byte) { var pos uint = 0 var from_pos uint = offset & mask - var i uint - for i = 0; i < num_commands; i++ { + for i := range cmds { var insert_len uint = uint(cmds[i].insert_len_) if from_pos+insert_len > mask { var head_size uint = mask + 1 - from_pos @@ -90,24 +88,19 @@ const clustersPerBatch = 16 func initBlockSplit(self *blockSplit) { self.num_types = 0 self.num_blocks = 0 - self.types = nil - self.lengths = nil + self.types = self.types[:0] + self.lengths = self.lengths[:0] self.types_alloc_size = 0 self.lengths_alloc_size = 0 } -func destroyBlockSplit(self *blockSplit) { - self.types = nil - self.lengths = nil -} - -func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) { +func splitBlock(cmds []command, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) { { - var literals_count uint = countLiterals(cmds, num_commands) + var literals_count uint = countLiterals(cmds) var literals []byte = make([]byte, literals_count) /* Create a continuous array of literals. */ - copyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals) + copyLiteralsToByteArray(cmds, data, pos, mask, literals) /* Create the block split on the array of literals. Literal histograms have alphabet size 256. */ @@ -116,28 +109,26 @@ func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask u literals = nil } { - var insert_and_copy_codes []uint16 = make([]uint16, num_commands) + var insert_and_copy_codes []uint16 = make([]uint16, len(cmds)) /* Compute prefix codes for commands. */ - var i uint - for i = 0; i < num_commands; i++ { + for i := range cmds { insert_and_copy_codes[i] = cmds[i].cmd_prefix_ } /* Create the block split on the array of command prefixes. */ - splitByteVectorCommand(insert_and_copy_codes, num_commands, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split) + splitByteVectorCommand(insert_and_copy_codes, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split) /* TODO: reuse for distances? */ insert_and_copy_codes = nil } { - var distance_prefixes []uint16 = make([]uint16, num_commands) + var distance_prefixes []uint16 = make([]uint16, len(cmds)) var j uint = 0 /* Create a continuous array of distance prefixes. */ - var i uint - for i = 0; i < num_commands; i++ { + for i := range cmds { var cmd *command = &cmds[i] if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF diff --git a/vendor/github.com/andybalholm/brotli/block_splitter_command.go b/vendor/github.com/andybalholm/brotli/block_splitter_command.go index e505fe13..9dec13e4 100644 --- a/vendor/github.com/andybalholm/brotli/block_splitter_command.go +++ b/vendor/github.com/andybalholm/brotli/block_splitter_command.go @@ -372,7 +372,8 @@ func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids histogram_symbols = nil } -func splitByteVectorCommand(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { +func splitByteVectorCommand(data []uint16, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { + length := uint(len(data)) var data_size uint = histogramDataSizeCommand() var num_histograms uint = length/literals_per_histogram + 1 var histograms []histogramCommand diff --git a/vendor/github.com/andybalholm/brotli/brotli_bit_stream.go b/vendor/github.com/andybalholm/brotli/brotli_bit_stream.go index 395f6049..ee655298 100644 --- a/vendor/github.com/andybalholm/brotli/brotli_bit_stream.go +++ b/vendor/github.com/andybalholm/brotli/brotli_bit_stream.go @@ -1,15 +1,24 @@ package brotli -import "math" +import ( + "math" + "sync" +) const maxHuffmanTreeSize = (2*numCommandSymbols + 1) -/* The maximum size of Huffman dictionary for distances assuming that - NPOSTFIX = 0 and NDIRECT = 0. */ +/* +The maximum size of Huffman dictionary for distances assuming that + + NPOSTFIX = 0 and NDIRECT = 0. +*/ const maxSimpleDistanceAlphabetSize = 140 -/* Represents the range of values belonging to a prefix code: - [offset, offset + 2^nbits) */ +/* +Represents the range of values belonging to a prefix code: + + [offset, offset + 2^nbits) +*/ type prefixCodeRange struct { offset uint32 nbits uint32 @@ -93,9 +102,12 @@ func nextBlockTypeCode(calculator *blockTypeCodeCalculator, type_ byte) uint { return type_code } -/* |nibblesbits| represents the 2 bits to encode MNIBBLES (0-3) - REQUIRES: length > 0 - REQUIRES: length <= (1 << 24) */ +/* +|nibblesbits| represents the 2 bits to encode MNIBBLES (0-3) + + REQUIRES: length > 0 + REQUIRES: length <= (1 << 24) +*/ func encodeMlen(length uint, bits *uint64, numbits *uint, nibblesbits *uint64) { var lg uint if length == 1 { @@ -129,8 +141,11 @@ func storeCommandExtra(cmd *command, storage_ix *uint, storage []byte) { writeBits(uint(insnumextra+getCopyExtra(copycode)), bits, storage_ix, storage) } -/* Data structure that stores almost everything that is needed to encode each - block switch command. */ +/* +Data structure that stores almost everything that is needed to encode each + + block switch command. +*/ type blockSplitCode struct { type_code_calculator blockTypeCodeCalculator type_depths [maxBlockTypeSymbols]byte @@ -151,9 +166,12 @@ func storeVarLenUint8(n uint, storage_ix *uint, storage []byte) { } } -/* Stores the compressed meta-block header. - REQUIRES: length > 0 - REQUIRES: length <= (1 << 24) */ +/* +Stores the compressed meta-block header. + + REQUIRES: length > 0 + REQUIRES: length <= (1 << 24) +*/ func storeCompressedMetaBlockHeader(is_final_block bool, length uint, storage_ix *uint, storage []byte) { var lenbits uint64 var nlenbits uint @@ -183,9 +201,12 @@ func storeCompressedMetaBlockHeader(is_final_block bool, length uint, storage_ix } } -/* Stores the uncompressed meta-block header. - REQUIRES: length > 0 - REQUIRES: length <= (1 << 24) */ +/* +Stores the uncompressed meta-block header. + + REQUIRES: length > 0 + REQUIRES: length <= (1 << 24) +*/ func storeUncompressedMetaBlockHeader(length uint, storage_ix *uint, storage []byte) { var lenbits uint64 var nlenbits uint @@ -309,8 +330,11 @@ func storeSimpleHuffmanTree(depths []byte, symbols []uint, num_symbols uint, max } } -/* num = alphabet size - depths = symbol depths */ +/* +num = alphabet size + + depths = symbol depths +*/ func storeHuffmanTree(depths []byte, num uint, tree []huffmanTree, storage_ix *uint, storage []byte) { var huffman_tree [numCommandSymbols]byte var huffman_tree_extra_bits [numCommandSymbols]byte @@ -364,8 +388,11 @@ func storeHuffmanTree(depths []byte, num uint, tree []huffmanTree, storage_ix *u storeHuffmanTreeToBitMask(huffman_tree_size, huffman_tree[:], huffman_tree_extra_bits[:], code_length_bitdepth[:], code_length_bitdepth_symbols[:], storage_ix, storage) } -/* Builds a Huffman tree from histogram[0:length] into depth[0:length] and - bits[0:length] and stores the encoded tree to the bit stream. */ +/* +Builds a Huffman tree from histogram[0:length] into depth[0:length] and + + bits[0:length] and stores the encoded tree to the bit stream. +*/ func buildAndStoreHuffmanTree(histogram []uint32, histogram_length uint, alphabet_size uint, tree []huffmanTree, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { var count uint = 0 var s4 = [4]uint{0} @@ -411,10 +438,12 @@ func buildAndStoreHuffmanTree(histogram []uint32, histogram_length uint, alphabe } } -func sortHuffmanTree1(v0 *huffmanTree, v1 *huffmanTree) bool { +func sortHuffmanTree1(v0 huffmanTree, v1 huffmanTree) bool { return v0.total_count_ < v1.total_count_ } +var huffmanTreePool sync.Pool + func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_bits uint, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { var count uint = 0 var symbols = [4]uint{0} @@ -446,7 +475,13 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ } { var max_tree_size uint = 2*length + 1 - var tree []huffmanTree = make([]huffmanTree, max_tree_size) + tree, _ := huffmanTreePool.Get().(*[]huffmanTree) + if tree == nil || cap(*tree) < int(max_tree_size) { + tmp := make([]huffmanTree, max_tree_size) + tree = &tmp + } else { + *tree = (*tree)[:max_tree_size] + } var count_limit uint32 for count_limit = 1; ; count_limit *= 2 { var node int = 0 @@ -455,9 +490,9 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ l-- if histogram[l] != 0 { if histogram[l] >= count_limit { - initHuffmanTree(&tree[node:][0], histogram[l], -1, int16(l)) + initHuffmanTree(&(*tree)[node:][0], histogram[l], -1, int16(l)) } else { - initHuffmanTree(&tree[node:][0], count_limit, -1, int16(l)) + initHuffmanTree(&(*tree)[node:][0], count_limit, -1, int16(l)) } node++ @@ -471,7 +506,7 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ var j int = n + 1 var k int - sortHuffmanTreeItems(tree, uint(n), huffmanTreeComparator(sortHuffmanTree1)) + sortHuffmanTreeItems(*tree, uint(n), huffmanTreeComparator(sortHuffmanTree1)) /* The nodes are: [0, n): the sorted leaf nodes that we start with. @@ -482,15 +517,15 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ There will be (2n+1) elements at the end. */ initHuffmanTree(&sentinel, math.MaxUint32, -1, -1) - tree[node] = sentinel + (*tree)[node] = sentinel node++ - tree[node] = sentinel + (*tree)[node] = sentinel node++ for k = n - 1; k > 0; k-- { var left int var right int - if tree[i].total_count_ <= tree[j].total_count_ { + if (*tree)[i].total_count_ <= (*tree)[j].total_count_ { left = i i++ } else { @@ -498,7 +533,7 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ j++ } - if tree[i].total_count_ <= tree[j].total_count_ { + if (*tree)[i].total_count_ <= (*tree)[j].total_count_ { right = i i++ } else { @@ -507,17 +542,17 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ } /* The sentinel node becomes the parent node. */ - tree[node-1].total_count_ = tree[left].total_count_ + tree[right].total_count_ + (*tree)[node-1].total_count_ = (*tree)[left].total_count_ + (*tree)[right].total_count_ - tree[node-1].index_left_ = int16(left) - tree[node-1].index_right_or_value_ = int16(right) + (*tree)[node-1].index_left_ = int16(left) + (*tree)[node-1].index_right_or_value_ = int16(right) /* Add back the last sentinel node. */ - tree[node] = sentinel + (*tree)[node] = sentinel node++ } - if setDepth(2*n-1, tree, depth, 14) { + if setDepth(2*n-1, *tree, depth, 14) { /* We need to pack the Huffman tree in 14 bits. If this was not successful, add fake entities to the lowest values and retry. */ break @@ -525,7 +560,7 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ } } - tree = nil + huffmanTreePool.Put(tree) } convertBitDepthsToSymbols(depth, length, bits) @@ -612,6 +647,203 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ } } +func buildAndStoreHuffmanTreeFastBW(histogram []uint32, histogram_total uint, max_bits uint, depth []byte, bits []uint16, bw *bitWriter) { + var count uint = 0 + var symbols = [4]uint{0} + var length uint = 0 + var total uint = histogram_total + for total != 0 { + if histogram[length] != 0 { + if count < 4 { + symbols[count] = length + } + + count++ + total -= uint(histogram[length]) + } + + length++ + } + + if count <= 1 { + bw.writeBits(4, 1) + bw.writeBits(max_bits, uint64(symbols[0])) + depth[symbols[0]] = 0 + bits[symbols[0]] = 0 + return + } + + for i := 0; i < int(length); i++ { + depth[i] = 0 + } + { + var max_tree_size uint = 2*length + 1 + tree, _ := huffmanTreePool.Get().(*[]huffmanTree) + if tree == nil || cap(*tree) < int(max_tree_size) { + tmp := make([]huffmanTree, max_tree_size) + tree = &tmp + } else { + *tree = (*tree)[:max_tree_size] + } + var count_limit uint32 + for count_limit = 1; ; count_limit *= 2 { + var node int = 0 + var l uint + for l = length; l != 0; { + l-- + if histogram[l] != 0 { + if histogram[l] >= count_limit { + initHuffmanTree(&(*tree)[node:][0], histogram[l], -1, int16(l)) + } else { + initHuffmanTree(&(*tree)[node:][0], count_limit, -1, int16(l)) + } + + node++ + } + } + { + var n int = node + /* Points to the next leaf node. */ /* Points to the next non-leaf node. */ + var sentinel huffmanTree + var i int = 0 + var j int = n + 1 + var k int + + sortHuffmanTreeItems(*tree, uint(n), huffmanTreeComparator(sortHuffmanTree1)) + + /* The nodes are: + [0, n): the sorted leaf nodes that we start with. + [n]: we add a sentinel here. + [n + 1, 2n): new parent nodes are added here, starting from + (n+1). These are naturally in ascending order. + [2n]: we add a sentinel at the end as well. + There will be (2n+1) elements at the end. */ + initHuffmanTree(&sentinel, math.MaxUint32, -1, -1) + + (*tree)[node] = sentinel + node++ + (*tree)[node] = sentinel + node++ + + for k = n - 1; k > 0; k-- { + var left int + var right int + if (*tree)[i].total_count_ <= (*tree)[j].total_count_ { + left = i + i++ + } else { + left = j + j++ + } + + if (*tree)[i].total_count_ <= (*tree)[j].total_count_ { + right = i + i++ + } else { + right = j + j++ + } + + /* The sentinel node becomes the parent node. */ + (*tree)[node-1].total_count_ = (*tree)[left].total_count_ + (*tree)[right].total_count_ + + (*tree)[node-1].index_left_ = int16(left) + (*tree)[node-1].index_right_or_value_ = int16(right) + + /* Add back the last sentinel node. */ + (*tree)[node] = sentinel + node++ + } + + if setDepth(2*n-1, *tree, depth, 14) { + /* We need to pack the Huffman tree in 14 bits. If this was not + successful, add fake entities to the lowest values and retry. */ + break + } + } + } + + huffmanTreePool.Put(tree) + } + + convertBitDepthsToSymbols(depth, length, bits) + if count <= 4 { + var i uint + + /* value of 1 indicates a simple Huffman code */ + bw.writeBits(2, 1) + + bw.writeBits(2, uint64(count)-1) /* NSYM - 1 */ + + /* Sort */ + for i = 0; i < count; i++ { + var j uint + for j = i + 1; j < count; j++ { + if depth[symbols[j]] < depth[symbols[i]] { + var tmp uint = symbols[j] + symbols[j] = symbols[i] + symbols[i] = tmp + } + } + } + + if count == 2 { + bw.writeBits(max_bits, uint64(symbols[0])) + bw.writeBits(max_bits, uint64(symbols[1])) + } else if count == 3 { + bw.writeBits(max_bits, uint64(symbols[0])) + bw.writeBits(max_bits, uint64(symbols[1])) + bw.writeBits(max_bits, uint64(symbols[2])) + } else { + bw.writeBits(max_bits, uint64(symbols[0])) + bw.writeBits(max_bits, uint64(symbols[1])) + bw.writeBits(max_bits, uint64(symbols[2])) + bw.writeBits(max_bits, uint64(symbols[3])) + + /* tree-select */ + bw.writeSingleBit(depth[symbols[0]] == 1) + } + } else { + var previous_value byte = 8 + var i uint + + /* Complex Huffman Tree */ + storeStaticCodeLengthCodeBW(bw) + + /* Actual RLE coding. */ + for i = 0; i < length; { + var value byte = depth[i] + var reps uint = 1 + var k uint + for k = i + 1; k < length && depth[k] == value; k++ { + reps++ + } + + i += reps + if value == 0 { + bw.writeBits(uint(kZeroRepsDepth[reps]), kZeroRepsBits[reps]) + } else { + if previous_value != value { + bw.writeBits(uint(kCodeLengthDepth[value]), uint64(kCodeLengthBits[value])) + reps-- + } + + if reps < 3 { + for reps != 0 { + reps-- + bw.writeBits(uint(kCodeLengthDepth[value]), uint64(kCodeLengthBits[value])) + } + } else { + reps -= 3 + bw.writeBits(uint(kNonZeroRepsDepth[reps]), kNonZeroRepsBits[reps]) + } + + previous_value = value + } + } + } +} + func indexOf(v []byte, v_size uint, value byte) uint { var i uint = 0 for ; i < v_size; i++ { @@ -663,12 +895,15 @@ func moveToFrontTransform(v_in []uint32, v_size uint, v_out []uint32) { } } -/* Finds runs of zeros in v[0..in_size) and replaces them with a prefix code of - the run length plus extra bits (lower 9 bits is the prefix code and the rest - are the extra bits). Non-zero values in v[] are shifted by - *max_length_prefix. Will not create prefix codes bigger than the initial - value of *max_run_length_prefix. The prefix code of run length L is simply - Log2Floor(L) and the number of extra bits is the same as the prefix code. */ +/* +Finds runs of zeros in v[0..in_size) and replaces them with a prefix code of + + the run length plus extra bits (lower 9 bits is the prefix code and the rest + are the extra bits). Non-zero values in v[] are shifted by + *max_length_prefix. Will not create prefix codes bigger than the initial + value of *max_run_length_prefix. The prefix code of run length L is simply + Log2Floor(L) and the number of extra bits is the same as the prefix code. +*/ func runLengthCodeZeros(in_size uint, v []uint32, out_size *uint, max_run_length_prefix *uint32) { var max_reps uint32 = 0 var i uint @@ -788,8 +1023,11 @@ func storeBlockSwitch(code *blockSplitCode, block_len uint32, block_type byte, i writeBits(uint(len_nextra), uint64(len_extra), storage_ix, storage) } -/* Builds a BlockSplitCode data structure from the block split given by the - vector of block types and block lengths and stores it to the bit stream. */ +/* +Builds a BlockSplitCode data structure from the block split given by the + + vector of block types and block lengths and stores it to the bit stream. +*/ func buildAndStoreBlockSplitCode(types []byte, lengths []uint32, num_blocks uint, num_types uint, tree []huffmanTree, code *blockSplitCode, storage_ix *uint, storage []byte) { var type_histo [maxBlockTypeSymbols]uint32 var length_histo [numBlockLenSymbols]uint32 @@ -875,37 +1113,53 @@ type blockEncoder struct { bits_ []uint16 } -func initBlockEncoder(self *blockEncoder, histogram_length uint, num_block_types uint, block_types []byte, block_lengths []uint32, num_blocks uint) { +var blockEncoderPool sync.Pool + +func getBlockEncoder(histogram_length uint, num_block_types uint, block_types []byte, block_lengths []uint32, num_blocks uint) *blockEncoder { + self, _ := blockEncoderPool.Get().(*blockEncoder) + + if self != nil { + self.block_ix_ = 0 + self.entropy_ix_ = 0 + self.depths_ = self.depths_[:0] + self.bits_ = self.bits_[:0] + } else { + self = &blockEncoder{} + } + self.histogram_length_ = histogram_length self.num_block_types_ = num_block_types self.block_types_ = block_types self.block_lengths_ = block_lengths self.num_blocks_ = num_blocks initBlockTypeCodeCalculator(&self.block_split_code_.type_code_calculator) - self.block_ix_ = 0 if num_blocks == 0 { self.block_len_ = 0 } else { self.block_len_ = uint(block_lengths[0]) } - self.entropy_ix_ = 0 - self.depths_ = nil - self.bits_ = nil + + return self } func cleanupBlockEncoder(self *blockEncoder) { - self.depths_ = nil - self.bits_ = nil + blockEncoderPool.Put(self) } -/* Creates entropy codes of block lengths and block types and stores them - to the bit stream. */ +/* +Creates entropy codes of block lengths and block types and stores them + + to the bit stream. +*/ func buildAndStoreBlockSwitchEntropyCodes(self *blockEncoder, tree []huffmanTree, storage_ix *uint, storage []byte) { buildAndStoreBlockSplitCode(self.block_types_, self.block_lengths_, self.num_blocks_, self.num_block_types_, tree, &self.block_split_code_, storage_ix, storage) } -/* Stores the next symbol with the entropy code of the current block type. - Updates the block type and block length at block boundaries. */ +/* +Stores the next symbol with the entropy code of the current block type. + + Updates the block type and block length at block boundaries. +*/ func storeSymbol(self *blockEncoder, symbol uint, storage_ix *uint, storage []byte) { if self.block_len_ == 0 { self.block_ix_++ @@ -924,9 +1178,12 @@ func storeSymbol(self *blockEncoder, symbol uint, storage_ix *uint, storage []by } } -/* Stores the next symbol with the entropy code of the current block type and - context value. - Updates the block type and block length at block boundaries. */ +/* +Stores the next symbol with the entropy code of the current block type and + + context value. + Updates the block type and block length at block boundaries. +*/ func storeSymbolWithContext(self *blockEncoder, symbol uint, context uint, context_map []uint32, storage_ix *uint, storage []byte, context_bits uint) { if self.block_len_ == 0 { self.block_ix_++ @@ -948,8 +1205,16 @@ func storeSymbolWithContext(self *blockEncoder, symbol uint, context uint, conte func buildAndStoreEntropyCodesLiteral(self *blockEncoder, histograms []histogramLiteral, histograms_size uint, alphabet_size uint, tree []huffmanTree, storage_ix *uint, storage []byte) { var table_size uint = histograms_size * self.histogram_length_ - self.depths_ = make([]byte, table_size) - self.bits_ = make([]uint16, table_size) + if cap(self.depths_) < int(table_size) { + self.depths_ = make([]byte, table_size) + } else { + self.depths_ = self.depths_[:table_size] + } + if cap(self.bits_) < int(table_size) { + self.bits_ = make([]uint16, table_size) + } else { + self.bits_ = self.bits_[:table_size] + } { var i uint for i = 0; i < histograms_size; i++ { @@ -961,8 +1226,16 @@ func buildAndStoreEntropyCodesLiteral(self *blockEncoder, histograms []histogram func buildAndStoreEntropyCodesCommand(self *blockEncoder, histograms []histogramCommand, histograms_size uint, alphabet_size uint, tree []huffmanTree, storage_ix *uint, storage []byte) { var table_size uint = histograms_size * self.histogram_length_ - self.depths_ = make([]byte, table_size) - self.bits_ = make([]uint16, table_size) + if cap(self.depths_) < int(table_size) { + self.depths_ = make([]byte, table_size) + } else { + self.depths_ = self.depths_[:table_size] + } + if cap(self.bits_) < int(table_size) { + self.bits_ = make([]uint16, table_size) + } else { + self.bits_ = self.bits_[:table_size] + } { var i uint for i = 0; i < histograms_size; i++ { @@ -974,8 +1247,16 @@ func buildAndStoreEntropyCodesCommand(self *blockEncoder, histograms []histogram func buildAndStoreEntropyCodesDistance(self *blockEncoder, histograms []histogramDistance, histograms_size uint, alphabet_size uint, tree []huffmanTree, storage_ix *uint, storage []byte) { var table_size uint = histograms_size * self.histogram_length_ - self.depths_ = make([]byte, table_size) - self.bits_ = make([]uint16, table_size) + if cap(self.depths_) < int(table_size) { + self.depths_ = make([]byte, table_size) + } else { + self.depths_ = self.depths_[:table_size] + } + if cap(self.bits_) < int(table_size) { + self.bits_ = make([]uint16, table_size) + } else { + self.bits_ = self.bits_[:table_size] + } { var i uint for i = 0; i < histograms_size; i++ { @@ -990,16 +1271,13 @@ func jumpToByteBoundary(storage_ix *uint, storage []byte) { storage[*storage_ix>>3] = 0 } -func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, n_commands uint, mb *metaBlockSplit, storage_ix *uint, storage []byte) { +func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, mb *metaBlockSplit, storage_ix *uint, storage []byte) { var pos uint = start_pos var i uint var num_distance_symbols uint32 = params.dist.alphabet_size var num_effective_distance_symbols uint32 = num_distance_symbols var tree []huffmanTree var literal_context_lut contextLUT = getContextLUT(literal_context_mode) - var literal_enc blockEncoder - var command_enc blockEncoder - var distance_enc blockEncoder var dist *distanceParams = ¶ms.dist if params.large_window && num_effective_distance_symbols > numHistogramDistanceSymbols { num_effective_distance_symbols = numHistogramDistanceSymbols @@ -1008,13 +1286,13 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b storeCompressedMetaBlockHeader(is_last, length, storage_ix, storage) tree = make([]huffmanTree, maxHuffmanTreeSize) - initBlockEncoder(&literal_enc, numLiteralSymbols, mb.literal_split.num_types, mb.literal_split.types, mb.literal_split.lengths, mb.literal_split.num_blocks) - initBlockEncoder(&command_enc, numCommandSymbols, mb.command_split.num_types, mb.command_split.types, mb.command_split.lengths, mb.command_split.num_blocks) - initBlockEncoder(&distance_enc, uint(num_effective_distance_symbols), mb.distance_split.num_types, mb.distance_split.types, mb.distance_split.lengths, mb.distance_split.num_blocks) + literal_enc := getBlockEncoder(numLiteralSymbols, mb.literal_split.num_types, mb.literal_split.types, mb.literal_split.lengths, mb.literal_split.num_blocks) + command_enc := getBlockEncoder(numCommandSymbols, mb.command_split.num_types, mb.command_split.types, mb.command_split.lengths, mb.command_split.num_blocks) + distance_enc := getBlockEncoder(uint(num_effective_distance_symbols), mb.distance_split.num_types, mb.distance_split.types, mb.distance_split.lengths, mb.distance_split.num_blocks) - buildAndStoreBlockSwitchEntropyCodes(&literal_enc, tree, storage_ix, storage) - buildAndStoreBlockSwitchEntropyCodes(&command_enc, tree, storage_ix, storage) - buildAndStoreBlockSwitchEntropyCodes(&distance_enc, tree, storage_ix, storage) + buildAndStoreBlockSwitchEntropyCodes(literal_enc, tree, storage_ix, storage) + buildAndStoreBlockSwitchEntropyCodes(command_enc, tree, storage_ix, storage) + buildAndStoreBlockSwitchEntropyCodes(distance_enc, tree, storage_ix, storage) writeBits(2, uint64(dist.distance_postfix_bits), storage_ix, storage) writeBits(4, uint64(dist.num_direct_distance_codes)>>dist.distance_postfix_bits, storage_ix, storage) @@ -1034,20 +1312,19 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b encodeContextMap(mb.distance_context_map, mb.distance_context_map_size, mb.distance_histograms_size, tree, storage_ix, storage) } - buildAndStoreEntropyCodesLiteral(&literal_enc, mb.literal_histograms, mb.literal_histograms_size, numLiteralSymbols, tree, storage_ix, storage) - buildAndStoreEntropyCodesCommand(&command_enc, mb.command_histograms, mb.command_histograms_size, numCommandSymbols, tree, storage_ix, storage) - buildAndStoreEntropyCodesDistance(&distance_enc, mb.distance_histograms, mb.distance_histograms_size, uint(num_distance_symbols), tree, storage_ix, storage) + buildAndStoreEntropyCodesLiteral(literal_enc, mb.literal_histograms, mb.literal_histograms_size, numLiteralSymbols, tree, storage_ix, storage) + buildAndStoreEntropyCodesCommand(command_enc, mb.command_histograms, mb.command_histograms_size, numCommandSymbols, tree, storage_ix, storage) + buildAndStoreEntropyCodesDistance(distance_enc, mb.distance_histograms, mb.distance_histograms_size, uint(num_distance_symbols), tree, storage_ix, storage) tree = nil - for i = 0; i < n_commands; i++ { - var cmd command = commands[i] + for _, cmd := range commands { var cmd_code uint = uint(cmd.cmd_prefix_) - storeSymbol(&command_enc, cmd_code, storage_ix, storage) + storeSymbol(command_enc, cmd_code, storage_ix, storage) storeCommandExtra(&cmd, storage_ix, storage) if mb.literal_context_map_size == 0 { var j uint for j = uint(cmd.insert_len_); j != 0; j-- { - storeSymbol(&literal_enc, uint(input[pos&mask]), storage_ix, storage) + storeSymbol(literal_enc, uint(input[pos&mask]), storage_ix, storage) pos++ } } else { @@ -1055,7 +1332,7 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b for j = uint(cmd.insert_len_); j != 0; j-- { var context uint = uint(getContext(prev_byte, prev_byte2, literal_context_lut)) var literal byte = input[pos&mask] - storeSymbolWithContext(&literal_enc, uint(literal), context, mb.literal_context_map, storage_ix, storage, literalContextBits) + storeSymbolWithContext(literal_enc, uint(literal), context, mb.literal_context_map, storage_ix, storage, literalContextBits) prev_byte2 = prev_byte prev_byte = literal pos++ @@ -1071,10 +1348,10 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b var distnumextra uint32 = uint32(cmd.dist_prefix_) >> 10 var distextra uint64 = uint64(cmd.dist_extra_) if mb.distance_context_map_size == 0 { - storeSymbol(&distance_enc, dist_code, storage_ix, storage) + storeSymbol(distance_enc, dist_code, storage_ix, storage) } else { var context uint = uint(commandDistanceContext(&cmd)) - storeSymbolWithContext(&distance_enc, dist_code, context, mb.distance_context_map, storage_ix, storage, distanceContextBits) + storeSymbolWithContext(distance_enc, dist_code, context, mb.distance_context_map, storage_ix, storage, distanceContextBits) } writeBits(uint(distnumextra), distextra, storage_ix, storage) @@ -1082,19 +1359,17 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b } } - cleanupBlockEncoder(&distance_enc) - cleanupBlockEncoder(&command_enc) - cleanupBlockEncoder(&literal_enc) + cleanupBlockEncoder(distance_enc) + cleanupBlockEncoder(command_enc) + cleanupBlockEncoder(literal_enc) if is_last { jumpToByteBoundary(storage_ix, storage) } } -func buildHistograms(input []byte, start_pos uint, mask uint, commands []command, n_commands uint, lit_histo *histogramLiteral, cmd_histo *histogramCommand, dist_histo *histogramDistance) { +func buildHistograms(input []byte, start_pos uint, mask uint, commands []command, lit_histo *histogramLiteral, cmd_histo *histogramCommand, dist_histo *histogramDistance) { var pos uint = start_pos - var i uint - for i = 0; i < n_commands; i++ { - var cmd command = commands[i] + for _, cmd := range commands { var j uint histogramAddCommand(cmd_histo, uint(cmd.cmd_prefix_)) for j = uint(cmd.insert_len_); j != 0; j-- { @@ -1109,11 +1384,9 @@ func buildHistograms(input []byte, start_pos uint, mask uint, commands []command } } -func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands []command, n_commands uint, lit_depth []byte, lit_bits []uint16, cmd_depth []byte, cmd_bits []uint16, dist_depth []byte, dist_bits []uint16, storage_ix *uint, storage []byte) { +func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands []command, lit_depth []byte, lit_bits []uint16, cmd_depth []byte, cmd_bits []uint16, dist_depth []byte, dist_bits []uint16, storage_ix *uint, storage []byte) { var pos uint = start_pos - var i uint - for i = 0; i < n_commands; i++ { - var cmd command = commands[i] + for _, cmd := range commands { var cmd_code uint = uint(cmd.cmd_prefix_) var j uint writeBits(uint(cmd_depth[cmd_code]), uint64(cmd_bits[cmd_code]), storage_ix, storage) @@ -1135,7 +1408,7 @@ func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands } } -func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) { +func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, storage_ix *uint, storage []byte) { var lit_histo histogramLiteral var cmd_histo histogramCommand var dist_histo histogramDistance @@ -1154,7 +1427,7 @@ func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, histogramClearCommand(&cmd_histo) histogramClearDistance(&dist_histo) - buildHistograms(input, start_pos, mask, commands, n_commands, &lit_histo, &cmd_histo, &dist_histo) + buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo) writeBits(13, 0, storage_ix, storage) @@ -1163,13 +1436,13 @@ func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, buildAndStoreHuffmanTree(cmd_histo.data_[:], numCommandSymbols, numCommandSymbols, tree, cmd_depth[:], cmd_bits[:], storage_ix, storage) buildAndStoreHuffmanTree(dist_histo.data_[:], maxSimpleDistanceAlphabetSize, uint(num_distance_symbols), tree, dist_depth[:], dist_bits[:], storage_ix, storage) tree = nil - storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) + storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) if is_last { jumpToByteBoundary(storage_ix, storage) } } -func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) { +func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, storage_ix *uint, storage []byte) { var num_distance_symbols uint32 = params.dist.alphabet_size var distance_alphabet_bits uint32 = log2FloorNonZero(uint(num_distance_symbols-1)) + 1 @@ -1177,15 +1450,13 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is writeBits(13, 0, storage_ix, storage) - if n_commands <= 128 { + if len(commands) <= 128 { var histogram = [numLiteralSymbols]uint32{0} var pos uint = start_pos var num_literals uint = 0 - var i uint var lit_depth [numLiteralSymbols]byte var lit_bits [numLiteralSymbols]uint16 - for i = 0; i < n_commands; i++ { - var cmd command = commands[i] + for _, cmd := range commands { var j uint for j = uint(cmd.insert_len_); j != 0; j-- { histogram[input[pos&mask]]++ @@ -1201,7 +1472,7 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is storeStaticCommandHuffmanTree(storage_ix, storage) storeStaticDistanceHuffmanTree(storage_ix, storage) - storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], storage_ix, storage) + storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], storage_ix, storage) } else { var lit_histo histogramLiteral var cmd_histo histogramCommand @@ -1215,7 +1486,7 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is histogramClearLiteral(&lit_histo) histogramClearCommand(&cmd_histo) histogramClearDistance(&dist_histo) - buildHistograms(input, start_pos, mask, commands, n_commands, &lit_histo, &cmd_histo, &dist_histo) + buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo) buildAndStoreHuffmanTreeFast(lit_histo.data_[:], lit_histo.total_count_, /* max_bits = */ 8, lit_depth[:], lit_bits[:], storage_ix, storage) @@ -1225,7 +1496,7 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is buildAndStoreHuffmanTreeFast(dist_histo.data_[:], dist_histo.total_count_, /* max_bits = */ uint(distance_alphabet_bits), dist_depth[:], dist_bits[:], storage_ix, storage) - storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) + storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) } if is_last { @@ -1233,8 +1504,11 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is } } -/* This is for storing uncompressed blocks (simple raw storage of - bytes-as-bytes). */ +/* +This is for storing uncompressed blocks (simple raw storage of + + bytes-as-bytes). +*/ func storeUncompressedMetaBlock(is_final_block bool, input []byte, position uint, mask uint, len uint, storage_ix *uint, storage []byte) { var masked_pos uint = position & mask storeUncompressedMetaBlockHeader(uint(len), storage_ix, storage) diff --git a/vendor/github.com/andybalholm/brotli/cluster_command.go b/vendor/github.com/andybalholm/brotli/cluster_command.go index 7449751b..45b569bb 100644 --- a/vendor/github.com/andybalholm/brotli/cluster_command.go +++ b/vendor/github.com/andybalholm/brotli/cluster_command.go @@ -1,7 +1,5 @@ package brotli -import "math" - /* Copyright 2013 Google Inc. All Rights Reserved. Distributed under MIT license. @@ -164,163 +162,3 @@ func histogramBitCostDistanceCommand(histogram *histogramCommand, candidate *his return populationCostCommand(&tmp) - candidate.bit_cost_ } } - -/* Find the best 'out' histogram for each of the 'in' histograms. - When called, clusters[0..num_clusters) contains the unique values from - symbols[0..in_size), but this property is not preserved in this function. - Note: we assume that out[]->bit_cost_ is already up-to-date. */ -func histogramRemapCommand(in []histogramCommand, in_size uint, clusters []uint32, num_clusters uint, out []histogramCommand, symbols []uint32) { - var i uint - for i = 0; i < in_size; i++ { - var best_out uint32 - if i == 0 { - best_out = symbols[0] - } else { - best_out = symbols[i-1] - } - var best_bits float64 = histogramBitCostDistanceCommand(&in[i], &out[best_out]) - var j uint - for j = 0; j < num_clusters; j++ { - var cur_bits float64 = histogramBitCostDistanceCommand(&in[i], &out[clusters[j]]) - if cur_bits < best_bits { - best_bits = cur_bits - best_out = clusters[j] - } - } - - symbols[i] = best_out - } - - /* Recompute each out based on raw and symbols. */ - for i = 0; i < num_clusters; i++ { - histogramClearCommand(&out[clusters[i]]) - } - - for i = 0; i < in_size; i++ { - histogramAddHistogramCommand(&out[symbols[i]], &in[i]) - } -} - -/* Reorders elements of the out[0..length) array and changes values in - symbols[0..length) array in the following way: - * when called, symbols[] contains indexes into out[], and has N unique - values (possibly N < length) - * on return, symbols'[i] = f(symbols[i]) and - out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length, - where f is a bijection between the range of symbols[] and [0..N), and - the first occurrences of values in symbols'[i] come in consecutive - increasing order. - Returns N, the number of unique values in symbols[]. */ - -var histogramReindexCommand_kInvalidIndex uint32 = math.MaxUint32 - -func histogramReindexCommand(out []histogramCommand, symbols []uint32, length uint) uint { - var new_index []uint32 = make([]uint32, length) - var next_index uint32 - var tmp []histogramCommand - var i uint - for i = 0; i < length; i++ { - new_index[i] = histogramReindexCommand_kInvalidIndex - } - - next_index = 0 - for i = 0; i < length; i++ { - if new_index[symbols[i]] == histogramReindexCommand_kInvalidIndex { - new_index[symbols[i]] = next_index - next_index++ - } - } - - /* TODO: by using idea of "cycle-sort" we can avoid allocation of - tmp and reduce the number of copying by the factor of 2. */ - tmp = make([]histogramCommand, next_index) - - next_index = 0 - for i = 0; i < length; i++ { - if new_index[symbols[i]] == next_index { - tmp[next_index] = out[symbols[i]] - next_index++ - } - - symbols[i] = new_index[symbols[i]] - } - - new_index = nil - for i = 0; uint32(i) < next_index; i++ { - out[i] = tmp[i] - } - - tmp = nil - return uint(next_index) -} - -func clusterHistogramsCommand(in []histogramCommand, in_size uint, max_histograms uint, out []histogramCommand, out_size *uint, histogram_symbols []uint32) { - var cluster_size []uint32 = make([]uint32, in_size) - var clusters []uint32 = make([]uint32, in_size) - var num_clusters uint = 0 - var max_input_histograms uint = 64 - var pairs_capacity uint = max_input_histograms * max_input_histograms / 2 - var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1)) - var i uint - - /* For the first pass of clustering, we allow all pairs. */ - for i = 0; i < in_size; i++ { - cluster_size[i] = 1 - } - - for i = 0; i < in_size; i++ { - out[i] = in[i] - out[i].bit_cost_ = populationCostCommand(&in[i]) - histogram_symbols[i] = uint32(i) - } - - for i = 0; i < in_size; i += max_input_histograms { - var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms) - var num_new_clusters uint - var j uint - for j = 0; j < num_to_combine; j++ { - clusters[num_clusters+j] = uint32(i + j) - } - - num_new_clusters = histogramCombineCommand(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity) - num_clusters += num_new_clusters - } - { - /* For the second pass, we limit the total number of histogram pairs. - After this limit is reached, we only keep searching for the best pair. */ - var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) - if pairs_capacity < (max_num_pairs + 1) { - var _new_size uint - if pairs_capacity == 0 { - _new_size = max_num_pairs + 1 - } else { - _new_size = pairs_capacity - } - var new_array []histogramPair - for _new_size < (max_num_pairs + 1) { - _new_size *= 2 - } - new_array = make([]histogramPair, _new_size) - if pairs_capacity != 0 { - copy(new_array, pairs[:pairs_capacity]) - } - - pairs = new_array - pairs_capacity = _new_size - } - - /* Collapse similar histograms. */ - num_clusters = histogramCombineCommand(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs) - } - - pairs = nil - cluster_size = nil - - /* Find the optimal map from original histograms to the final ones. */ - histogramRemapCommand(in, in_size, clusters, num_clusters, out, histogram_symbols) - - clusters = nil - - /* Convert the context map to a canonical form. */ - *out_size = histogramReindexCommand(out, histogram_symbols, in_size) -} diff --git a/vendor/github.com/andybalholm/brotli/command.go b/vendor/github.com/andybalholm/brotli/command.go index e93ccdfa..b1662a55 100644 --- a/vendor/github.com/andybalholm/brotli/command.go +++ b/vendor/github.com/andybalholm/brotli/command.go @@ -194,26 +194,28 @@ type command struct { } /* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */ -func initCommand(self *command, dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) { +func makeCommand(dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) (cmd command) { /* Don't rely on signed int representation, use honest casts. */ var delta uint32 = uint32(byte(int8(copylen_code_delta))) - self.insert_len_ = uint32(insertlen) - self.copy_len_ = uint32(uint32(copylen) | delta<<25) + cmd.insert_len_ = uint32(insertlen) + cmd.copy_len_ = uint32(uint32(copylen) | delta<<25) /* The distance prefix and extra bits are stored in this Command as if npostfix and ndirect were 0, they are only recomputed later after the clustering if needed. */ - prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &self.dist_prefix_, &self.dist_extra_) + prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_) + getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (cmd.dist_prefix_&0x3FF == 0), &cmd.cmd_prefix_) - getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (self.dist_prefix_&0x3FF == 0), &self.cmd_prefix_) + return cmd } -func initInsertCommand(self *command, insertlen uint) { - self.insert_len_ = uint32(insertlen) - self.copy_len_ = 4 << 25 - self.dist_extra_ = 0 - self.dist_prefix_ = numDistanceShortCodes - getLengthCode(insertlen, 4, false, &self.cmd_prefix_) +func makeInsertCommand(insertlen uint) (cmd command) { + cmd.insert_len_ = uint32(insertlen) + cmd.copy_len_ = 4 << 25 + cmd.dist_extra_ = 0 + cmd.dist_prefix_ = numDistanceShortCodes + getLengthCode(insertlen, 4, false, &cmd.cmd_prefix_) + return cmd } func commandRestoreDistanceCode(self *command, dist *distanceParams) uint32 { diff --git a/vendor/github.com/andybalholm/brotli/compress_fragment.go b/vendor/github.com/andybalholm/brotli/compress_fragment.go index 435898e1..c9bd0577 100644 --- a/vendor/github.com/andybalholm/brotli/compress_fragment.go +++ b/vendor/github.com/andybalholm/brotli/compress_fragment.go @@ -33,14 +33,8 @@ func hashBytesAtOffset5(v uint64, offset int, shift uint) uint32 { } func isMatch5(p1 []byte, p2 []byte) bool { - var i int - for i = 0; i < 5; i++ { - if p1[i] != p2[i] { - return false - } - } - - return true + return binary.LittleEndian.Uint32(p1) == binary.LittleEndian.Uint32(p2) && + p1[4] == p2[4] } /* Builds a literal prefix code into "depths" and "bits" based on the statistics @@ -610,7 +604,7 @@ emit_commands: assert(candidate < ip) table[hash] = int(ip - base_ip) - if !(!isMatch5(in[ip:], in[candidate:])) { + if isMatch5(in[ip:], in[candidate:]) { break } } diff --git a/vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go b/vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go index ffeb3216..79f9c7fd 100644 --- a/vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +++ b/vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go @@ -30,18 +30,20 @@ func hashBytesAtOffset(v uint64, offset uint, shift uint, length uint) uint32 { } func isMatch1(p1 []byte, p2 []byte, length uint) bool { - var i uint - for i = 0; i < length && i < 6; i++ { - if p1[i] != p2[i] { - return false - } + if binary.LittleEndian.Uint32(p1) != binary.LittleEndian.Uint32(p2) { + return false } - - return true + if length == 4 { + return true + } + return p1[4] == p2[4] && p1[5] == p2[5] } -/* Builds a command and distance prefix code (each 64 symbols) into "depth" and - "bits" based on "histogram" and stores it into the bit stream. */ +/* +Builds a command and distance prefix code (each 64 symbols) into "depth" and + + "bits" based on "histogram" and stores it into the bit stream. +*/ func buildAndStoreCommandPrefixCode(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { var tree [129]huffmanTree var cmd_depth = [numCommandSymbols]byte{0} @@ -217,6 +219,25 @@ func storeMetaBlockHeader(len uint, is_uncompressed bool, storage_ix *uint, stor writeSingleBit(is_uncompressed, storage_ix, storage) } +func storeMetaBlockHeaderBW(len uint, is_uncompressed bool, bw *bitWriter) { + var nibbles uint = 6 + + /* ISLAST */ + bw.writeBits(1, 0) + + if len <= 1<<16 { + nibbles = 4 + } else if len <= 1<<20 { + nibbles = 5 + } + + bw.writeBits(2, uint64(nibbles)-4) + bw.writeBits(nibbles*4, uint64(len)-1) + + /* ISUNCOMPRESSED */ + bw.writeSingleBit(is_uncompressed) +} + func createCommands(input []byte, block_size uint, input_size uint, base_ip_ptr []byte, table []int, table_bits uint, min_match uint, literals *[]byte, commands *[]uint32) { var ip int = 0 var shift uint = 64 - table_bits @@ -711,19 +732,22 @@ func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, co } } -/* Compresses "input" string to the "*storage" buffer as one or more complete - meta-blocks, and updates the "*storage_ix" bit position. +/* +Compresses "input" string to the "*storage" buffer as one or more complete - If "is_last" is 1, emits an additional empty last meta-block. + meta-blocks, and updates the "*storage_ix" bit position. - REQUIRES: "input_size" is greater than zero, or "is_last" is 1. - REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24). - REQUIRES: "command_buf" and "literal_buf" point to at least - kCompressFragmentTwoPassBlockSize long arrays. - REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. - REQUIRES: "table_size" is a power of two - OUTPUT: maximal copy distance <= |input_size| - OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */ + If "is_last" is 1, emits an additional empty last meta-block. + + REQUIRES: "input_size" is greater than zero, or "is_last" is 1. + REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24). + REQUIRES: "command_buf" and "literal_buf" point to at least + kCompressFragmentTwoPassBlockSize long arrays. + REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. + REQUIRES: "table_size" is a power of two + OUTPUT: maximal copy distance <= |input_size| + OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) +*/ func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, storage_ix *uint, storage []byte) { var initial_storage_ix uint = *storage_ix var table_bits uint = uint(log2FloorNonZero(table_size)) diff --git a/vendor/github.com/andybalholm/brotli/decode.go b/vendor/github.com/andybalholm/brotli/decode.go index d2f39a05..9d9513b7 100644 --- a/vendor/github.com/andybalholm/brotli/decode.go +++ b/vendor/github.com/andybalholm/brotli/decode.go @@ -50,21 +50,6 @@ const ( decoderErrorUnreachable = -31 ) -/** - * The value of the last error code, negative integer. - * - * All other error code values are in the range from ::lastErrorCode - * to @c -1. There are also 4 other possible non-error codes @c 0 .. @c 3 in - * ::BrotliDecoderErrorCode enumeration. - */ -const lastErrorCode = decoderErrorUnreachable - -/** Options to be used with ::BrotliDecoderSetParameter. */ -const ( - decoderParamDisableRingBufferReallocation = 0 - decoderParamLargeWindow = 1 -) - const huffmanTableBits = 8 const huffmanTableMask = 0xFF @@ -81,28 +66,6 @@ var kCodeLengthPrefixLength = [16]byte{2, 2, 2, 3, 2, 2, 2, 4, 2, 2, 2, 3, 2, 2, var kCodeLengthPrefixValue = [16]byte{0, 4, 3, 2, 0, 4, 3, 1, 0, 4, 3, 2, 0, 4, 3, 5} -func decoderSetParameter(state *Reader, p int, value uint32) bool { - if state.state != stateUninited { - return false - } - switch p { - case decoderParamDisableRingBufferReallocation: - if !(value == 0) { - state.canny_ringbuffer_allocation = 0 - } else { - state.canny_ringbuffer_allocation = 1 - } - return true - - case decoderParamLargeWindow: - state.large_window = (!(value == 0)) - return true - - default: - return false - } -} - /* Saves error code and converts it to BrotliDecoderResult. */ func saveErrorCode(s *Reader, e int) int { s.error_code = int(e) @@ -1125,10 +1088,8 @@ func decodeContextMap(context_map_size uint32, num_htrees *uint32, context_map_a Reads 3..54 bits. */ func decodeBlockTypeAndLength(safe int, s *Reader, tree_type int) bool { var max_block_type uint32 = s.num_block_types[tree_type] - var type_tree []huffmanCode - type_tree = s.block_type_trees[tree_type*huffmanMaxSize258:] - var len_tree []huffmanCode - len_tree = s.block_len_trees[tree_type*huffmanMaxSize26:] + type_tree := s.block_type_trees[tree_type*huffmanMaxSize258:] + len_tree := s.block_len_trees[tree_type*huffmanMaxSize26:] var br *bitReader = &s.br var ringbuffer []uint32 = s.block_type_rb[tree_type*2:] var block_type uint32 @@ -1280,8 +1241,7 @@ func unwrittenBytes(s *Reader, wrap bool) uint { Returns BROTLI_DECODER_NEEDS_MORE_OUTPUT only if there is more output to push and either ring-buffer is as big as window size, or |force| is true. */ func writeRingBuffer(s *Reader, available_out *uint, next_out *[]byte, total_out *uint, force bool) int { - var start []byte - start = s.ringbuffer[s.partial_pos_out&uint(s.ringbuffer_mask):] + start := s.ringbuffer[s.partial_pos_out&uint(s.ringbuffer_mask):] var to_write uint = unwrittenBytes(s, true) var num_written uint = *available_out if num_written > to_write { @@ -1344,26 +1304,21 @@ func wrapRingBuffer(s *Reader) { Last two bytes of ring-buffer are initialized to 0, so context calculation could be done uniformly for the first two and all other positions. */ func ensureRingBuffer(s *Reader) bool { - var old_ringbuffer []byte = s.ringbuffer + var old_ringbuffer []byte if s.ringbuffer_size == s.new_ringbuffer_size { return true } - - s.ringbuffer = make([]byte, uint(s.new_ringbuffer_size)+uint(kRingBufferWriteAheadSlack)) - if s.ringbuffer == nil { - /* Restore previous value. */ - s.ringbuffer = old_ringbuffer - - return false + spaceNeeded := int(s.new_ringbuffer_size) + int(kRingBufferWriteAheadSlack) + if len(s.ringbuffer) < spaceNeeded { + old_ringbuffer = s.ringbuffer + s.ringbuffer = make([]byte, spaceNeeded) } s.ringbuffer[s.new_ringbuffer_size-2] = 0 s.ringbuffer[s.new_ringbuffer_size-1] = 0 - if !(old_ringbuffer == nil) { + if old_ringbuffer != nil { copy(s.ringbuffer, old_ringbuffer[:uint(s.pos)]) - - old_ringbuffer = nil } s.ringbuffer_size = s.new_ringbuffer_size @@ -1412,8 +1367,7 @@ func copyUncompressedBlockToOutput(available_out *uint, next_out *[]byte, total_ case stateUncompressedWrite: { - var result int - result = writeRingBuffer(s, available_out, next_out, total_out, false) + result := writeRingBuffer(s, available_out, next_out, total_out, false) if result != decoderSuccess { return result } @@ -1931,8 +1885,7 @@ CommandPostDecodeLiterals: } if transform_idx < int(trans.num_transforms) { - var word []byte - word = words.data[offset:] + word := words.data[offset:] var len int = i if transform_idx == int(trans.cutOffTransforms[0]) { copy(s.ringbuffer[pos:], word[:uint(len)]) @@ -1954,10 +1907,8 @@ CommandPostDecodeLiterals: } } else { var src_start int = (pos - s.distance_code) & s.ringbuffer_mask - var copy_dst []byte - copy_dst = s.ringbuffer[pos:] - var copy_src []byte - copy_src = s.ringbuffer[src_start:] + copy_dst := s.ringbuffer[pos:] + copy_src := s.ringbuffer[src_start:] var dst_end int = pos + i var src_end int = src_start + i @@ -2494,8 +2445,6 @@ func decoderDecompressStream(s *Reader, available_in *uint, next_in *[]byte, ava } else { s.state = stateCommandBegin } - - break } else if s.state == stateCommandPostWrite2 { s.state = stateCommandPostWrapCopy /* BROTLI_STATE_COMMAND_INNER_WRITE */ } else { diff --git a/vendor/github.com/andybalholm/brotli/encode.go b/vendor/github.com/andybalholm/brotli/encode.go index 0dd3a3e1..8e25a4ec 100644 --- a/vendor/github.com/andybalholm/brotli/encode.go +++ b/vendor/github.com/andybalholm/brotli/encode.go @@ -72,15 +72,15 @@ const ( ) type Writer struct { - dst io.Writer + dst io.Writer + options WriterOptions + err error params encoderParams hasher_ hasherHandle input_pos_ uint64 ringbuffer_ ringBuffer - cmd_alloc_size_ uint - commands_ []command - num_commands_ uint + commands []command num_literals_ uint last_insert_len_ uint last_flush_pos_ uint64 @@ -91,8 +91,7 @@ type Writer struct { last_bytes_bits_ byte prev_byte_ byte prev_byte2_ byte - storage_size_ uint - storage_ []byte + storage []byte small_table_ [1 << 10]int large_table_ []int large_table_size_ uint @@ -102,9 +101,6 @@ type Writer struct { cmd_code_numbits_ uint command_buf_ []uint32 literal_buf_ []byte - next_out_ []byte - available_out_ uint - total_out_ uint tiny_buf_ struct { u64 [2]uint64 u8 [16]byte @@ -145,14 +141,12 @@ func wrapPosition(position uint64) uint32 { return result } -func getBrotliStorage(s *Writer, size uint) []byte { - if s.storage_size_ < size { - s.storage_ = nil - s.storage_ = make([]byte, size) - s.storage_size_ = size +func (s *Writer) getStorage(size int) []byte { + if len(s.storage) < size { + s.storage = make([]byte, size) } - return s.storage_ + return s.storage } func hashTableSize(max_table_size uint, input_size uint) uint { @@ -221,332 +215,6 @@ func encodeWindowBits(lgwin int, large_window bool, last_bytes *uint16, last_byt } } -/* Initializes the command and distance prefix codes for the first block. */ - -var initCommandPrefixCodes_kDefaultCommandDepths = [128]byte{ - 0, - 4, - 4, - 5, - 6, - 6, - 7, - 7, - 7, - 7, - 7, - 8, - 8, - 8, - 8, - 8, - 0, - 0, - 0, - 4, - 4, - 4, - 4, - 4, - 5, - 5, - 6, - 6, - 6, - 6, - 7, - 7, - 7, - 7, - 10, - 10, - 10, - 10, - 10, - 10, - 0, - 4, - 4, - 5, - 5, - 5, - 6, - 6, - 7, - 8, - 8, - 9, - 10, - 10, - 10, - 10, - 10, - 10, - 10, - 10, - 10, - 10, - 10, - 10, - 5, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 6, - 6, - 6, - 6, - 6, - 6, - 5, - 5, - 5, - 5, - 5, - 5, - 4, - 4, - 4, - 4, - 4, - 4, - 4, - 5, - 5, - 5, - 5, - 5, - 5, - 6, - 6, - 7, - 7, - 7, - 8, - 10, - 12, - 12, - 12, - 12, - 12, - 12, - 12, - 12, - 12, - 12, - 12, - 12, -} -var initCommandPrefixCodes_kDefaultCommandBits = [128]uint16{ - 0, - 0, - 8, - 9, - 3, - 35, - 7, - 71, - 39, - 103, - 23, - 47, - 175, - 111, - 239, - 31, - 0, - 0, - 0, - 4, - 12, - 2, - 10, - 6, - 13, - 29, - 11, - 43, - 27, - 59, - 87, - 55, - 15, - 79, - 319, - 831, - 191, - 703, - 447, - 959, - 0, - 14, - 1, - 25, - 5, - 21, - 19, - 51, - 119, - 159, - 95, - 223, - 479, - 991, - 63, - 575, - 127, - 639, - 383, - 895, - 255, - 767, - 511, - 1023, - 14, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 27, - 59, - 7, - 39, - 23, - 55, - 30, - 1, - 17, - 9, - 25, - 5, - 0, - 8, - 4, - 12, - 2, - 10, - 6, - 21, - 13, - 29, - 3, - 19, - 11, - 15, - 47, - 31, - 95, - 63, - 127, - 255, - 767, - 2815, - 1791, - 3839, - 511, - 2559, - 1535, - 3583, - 1023, - 3071, - 2047, - 4095, -} -var initCommandPrefixCodes_kDefaultCommandCode = []byte{ - 0xff, - 0x77, - 0xd5, - 0xbf, - 0xe7, - 0xde, - 0xea, - 0x9e, - 0x51, - 0x5d, - 0xde, - 0xc6, - 0x70, - 0x57, - 0xbc, - 0x58, - 0x58, - 0x58, - 0xd8, - 0xd8, - 0x58, - 0xd5, - 0xcb, - 0x8c, - 0xea, - 0xe0, - 0xc3, - 0x87, - 0x1f, - 0x83, - 0xc1, - 0x60, - 0x1c, - 0x67, - 0xb2, - 0xaa, - 0x06, - 0x83, - 0xc1, - 0x60, - 0x30, - 0x18, - 0xcc, - 0xa1, - 0xce, - 0x88, - 0x54, - 0x94, - 0x46, - 0xe1, - 0xb0, - 0xd0, - 0x4e, - 0xb2, - 0xf7, - 0x04, - 0x00, -} -var initCommandPrefixCodes_kDefaultCommandCodeNumBits uint = 448 - -func initCommandPrefixCodes(cmd_depths []byte, cmd_bits []uint16, cmd_code []byte, cmd_code_numbits *uint) { - copy(cmd_depths, initCommandPrefixCodes_kDefaultCommandDepths[:]) - copy(cmd_bits, initCommandPrefixCodes_kDefaultCommandBits[:]) - - /* Initialize the pre-compressed form of the command and distance prefix - codes. */ - copy(cmd_code, initCommandPrefixCodes_kDefaultCommandCode) - - *cmd_code_numbits = initCommandPrefixCodes_kDefaultCommandCodeNumBits -} - /* Decide about the context map based on the ability of the prediction ability of the previous byte UTF8-prefix on the next byte. The prediction ability is calculated as Shannon entropy. Here we need @@ -556,136 +224,16 @@ func initCommandPrefixCodes(cmd_depths []byte, cmd_bits []uint16, cmd_code []byt coding. */ var kStaticContextMapContinuation = [64]uint32{ - 1, - 1, - 2, - 2, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, + 1, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } var kStaticContextMapSimpleUTF8 = [64]uint32{ - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, + 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } func chooseContextMap(quality int, bigram_histo []uint32, num_literal_contexts *uint, literal_context_map *[]uint32) { @@ -737,70 +285,22 @@ func chooseContextMap(quality int, bigram_histo []uint32, num_literal_contexts * first 5 bits of literals. */ var kStaticContextMapComplexUTF8 = [64]uint32{ - 11, - 11, - 12, - 12, - 0, - 0, - 0, - 0, - 1, - 1, - 9, - 9, - 2, - 2, - 2, - 2, - 1, - 1, - 1, - 1, - 8, - 3, - 3, - 3, - 1, - 1, - 1, - 1, - 2, - 2, - 2, - 2, - 8, - 4, - 4, - 4, - 8, - 7, - 4, - 4, - 8, - 0, - 0, - 0, - 3, - 3, - 3, - 3, - 5, - 5, - 10, - 5, - 5, - 5, - 10, - 5, - 6, - 6, - 6, - 6, - 6, - 6, - 6, - 6, + 11, 11, 12, 12, /* 0 special */ + 0, 0, 0, 0, /* 4 lf */ + 1, 1, 9, 9, /* 8 space */ + 2, 2, 2, 2, /* !, first after space/lf and after something else. */ + 1, 1, 1, 1, /* " */ + 8, 3, 3, 3, /* % */ + 1, 1, 1, 1, /* ({[ */ + 2, 2, 2, 2, /* }]) */ + 8, 4, 4, 4, /* :; */ + 8, 7, 4, 4, /* . */ + 8, 0, 0, 0, /* > */ + 3, 3, 3, 3, /* [0..9] */ + 5, 5, 10, 5, /* [A-Z] */ + 5, 5, 10, 5, + 6, 6, 6, 6, /* [a-z] */ + 6, 6, 6, 6, } func shouldUseComplexStaticContextMap(input []byte, start_pos uint, length uint, mask uint, quality int, size_hint uint, num_literal_contexts *uint, literal_context_map *[]uint32) bool { @@ -932,7 +432,7 @@ func chooseContextMode(params *encoderParams, data []byte, pos uint, mask uint, return contextUTF8 } -func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, num_commands uint, commands []command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) { +func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, commands []command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) { var wrapped_last_flush_pos uint32 = wrapPosition(last_flush_pos) var last_bytes uint16 var last_bytes_bits byte @@ -947,7 +447,7 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes return } - if !shouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, num_commands) { + if !shouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, uint(len(commands))) { /* Restore the distance cache, as its last update by CreateBackwardReferences is now unused. */ copy(dist_cache, saved_dist_cache[:4]) @@ -960,12 +460,11 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes last_bytes = uint16(storage[1])<<8 | uint16(storage[0]) last_bytes_bits = byte(*storage_ix) if params.quality <= maxQualityForStaticEntropyCodes { - storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, num_commands, storage_ix, storage) + storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage) } else if params.quality < minQualityForBlockSplit { - storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, num_commands, storage_ix, storage) + storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage) } else { - var mb metaBlockSplit - initMetaBlockSplit(&mb) + mb := getMetaBlockSplit() if params.quality < minQualityForHqBlockSplitting { var num_literal_contexts uint = 1 var literal_context_map []uint32 = nil @@ -973,9 +472,9 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes decideOverLiteralContextModeling(data, uint(wrapped_last_flush_pos), bytes, mask, params.quality, params.size_hint, &num_literal_contexts, &literal_context_map) } - buildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, num_commands, &mb) + buildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, mb) } else { - buildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, num_commands, literal_context_mode, &mb) + buildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, literal_context_mode, mb) } if params.quality >= minQualityForOptimizeHistograms { @@ -987,11 +486,11 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes num_effective_dist_codes = numHistogramDistanceSymbols } - optimizeHistograms(num_effective_dist_codes, &mb) + optimizeHistograms(num_effective_dist_codes, mb) } - storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, num_commands, &mb, storage_ix, storage) - destroyMetaBlockSplit(&mb) + storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, mb, storage_ix, storage) + freeMetaBlockSplit(mb) } if bytes+4 < *storage_ix>>3 { @@ -1055,7 +554,38 @@ func ensureInitialized(s *Writer) bool { } if s.params.quality == fastOnePassCompressionQuality { - initCommandPrefixCodes(s.cmd_depths_[:], s.cmd_bits_[:], s.cmd_code_[:], &s.cmd_code_numbits_) + s.cmd_depths_ = [128]byte{ + 0, 4, 4, 5, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, + 0, 0, 0, 4, 4, 4, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, + 7, 7, 10, 10, 10, 10, 10, 10, 0, 4, 4, 5, 5, 5, 6, 6, + 7, 8, 8, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, + 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 7, 7, 7, 8, 10, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + } + s.cmd_bits_ = [128]uint16{ + 0, 0, 8, 9, 3, 35, 7, 71, + 39, 103, 23, 47, 175, 111, 239, 31, + 0, 0, 0, 4, 12, 2, 10, 6, + 13, 29, 11, 43, 27, 59, 87, 55, + 15, 79, 319, 831, 191, 703, 447, 959, + 0, 14, 1, 25, 5, 21, 19, 51, + 119, 159, 95, 223, 479, 991, 63, 575, + 127, 639, 383, 895, 255, 767, 511, 1023, + 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 27, 59, 7, 39, 23, 55, 30, 1, 17, 9, 25, 5, 0, 8, 4, 12, + 2, 10, 6, 21, 13, 29, 3, 19, 11, 15, 47, 31, 95, 63, 127, 255, + 767, 2815, 1791, 3839, 511, 2559, 1535, 3583, 1023, 3071, 2047, 4095, + } + s.cmd_code_ = [512]byte{ + 0xff, 0x77, 0xd5, 0xbf, 0xe7, 0xde, 0xea, 0x9e, 0x51, 0x5d, 0xde, 0xc6, + 0x70, 0x57, 0xbc, 0x58, 0x58, 0x58, 0xd8, 0xd8, 0x58, 0xd5, 0xcb, 0x8c, + 0xea, 0xe0, 0xc3, 0x87, 0x1f, 0x83, 0xc1, 0x60, 0x1c, 0x67, 0xb2, 0xaa, + 0x06, 0x83, 0xc1, 0x60, 0x30, 0x18, 0xcc, 0xa1, 0xce, 0x88, 0x54, 0x94, + 0x46, 0xe1, 0xb0, 0xd0, 0x4e, 0xb2, 0xf7, 0x04, 0x00, + } + s.cmd_code_numbits_ = 448 } s.is_initialized_ = true @@ -1080,33 +610,23 @@ func encoderInitParams(params *encoderParams) { func encoderInitState(s *Writer) { encoderInitParams(&s.params) s.input_pos_ = 0 - s.num_commands_ = 0 + s.commands = s.commands[:0] s.num_literals_ = 0 s.last_insert_len_ = 0 s.last_flush_pos_ = 0 s.last_processed_pos_ = 0 s.prev_byte_ = 0 s.prev_byte2_ = 0 - s.storage_size_ = 0 - s.storage_ = nil - s.hasher_ = nil - s.large_table_ = nil - s.large_table_size_ = 0 + if s.hasher_ != nil { + s.hasher_.Common().is_prepared_ = false + } s.cmd_code_numbits_ = 0 - s.command_buf_ = nil - s.literal_buf_ = nil - s.next_out_ = nil - s.available_out_ = 0 - s.total_out_ = 0 s.stream_state_ = streamProcessing s.is_last_block_emitted_ = false s.is_initialized_ = false ringBufferInit(&s.ringbuffer_) - s.commands_ = nil - s.cmd_alloc_size_ = 0 - /* Initialize distance cache. */ s.dist_cache_[0] = 4 @@ -1189,7 +709,7 @@ func updateLastProcessedPos(s *Writer) bool { } func extendLastCommand(s *Writer, bytes *uint32, wrapped_last_processed_pos *uint32) { - var last_command *command = &s.commands_[s.num_commands_-1] + var last_command *command = &s.commands[len(s.commands)-1] var data []byte = s.ringbuffer_.buffer_ var mask uint32 = s.ringbuffer_.mask_ var max_backward_distance uint64 = ((uint64(1)) << s.params.lgwin) - windowGap @@ -1218,18 +738,17 @@ func extendLastCommand(s *Writer, bytes *uint32, wrapped_last_processed_pos *uin } /* - Processes the accumulated input data and sets |*out_size| to the length of - the new output meta-block, or to zero if no new output meta-block has been - created (in this case the processed input data is buffered internally). - If |*out_size| is positive, |*output| points to the start of the output - data. If |is_last| or |force_flush| is true, an output meta-block is + Processes the accumulated input data and writes + the new output meta-block to s.dest, if one has been + created (otherwise the processed input data is buffered internally). + If |is_last| or |force_flush| is true, an output meta-block is always created. However, until |is_last| is true encoder may retain up to 7 bits of the last byte of output. To force encoder to dump the remaining bits use WriteMetadata() to append an empty meta-data block. Returns false if the size of the input data is larger than input_block_size(). */ -func encodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, output *[]byte) bool { +func encodeData(s *Writer, is_last bool, force_flush bool) bool { var delta uint64 = unprocessedInputSize(s) var bytes uint32 = uint32(delta) var wrapped_last_processed_pos uint32 = wrapPosition(s.last_processed_pos_) @@ -1252,9 +771,14 @@ func encodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, outpu return false } - if s.params.quality == fastTwoPassCompressionQuality && s.command_buf_ == nil { - s.command_buf_ = make([]uint32, kCompressFragmentTwoPassBlockSize) - s.literal_buf_ = make([]byte, kCompressFragmentTwoPassBlockSize) + if s.params.quality == fastTwoPassCompressionQuality { + if s.command_buf_ == nil || cap(s.command_buf_) < int(kCompressFragmentTwoPassBlockSize) { + s.command_buf_ = make([]uint32, kCompressFragmentTwoPassBlockSize) + s.literal_buf_ = make([]byte, kCompressFragmentTwoPassBlockSize) + } else { + s.command_buf_ = s.command_buf_[:kCompressFragmentTwoPassBlockSize] + s.literal_buf_ = s.literal_buf_[:kCompressFragmentTwoPassBlockSize] + } } if s.params.quality == fastOnePassCompressionQuality || s.params.quality == fastTwoPassCompressionQuality { @@ -1266,12 +790,10 @@ func encodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, outpu if delta == 0 && !is_last { /* We have no new input data and we don't have to finish the stream, so nothing to do. */ - *out_size = 0 - return true } - storage = getBrotliStorage(s, uint(2*bytes+503)) + storage = s.getStorage(int(2*bytes + 503)) storage[0] = byte(s.last_bytes_) storage[1] = byte(s.last_bytes_ >> 8) table = getHashTable(s, s.params.quality, uint(bytes), &table_size) @@ -1284,28 +806,23 @@ func encodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, outpu s.last_bytes_ = uint16(storage[storage_ix>>3]) s.last_bytes_bits_ = byte(storage_ix & 7) updateLastProcessedPos(s) - *output = storage[0:] - *out_size = storage_ix >> 3 + s.writeOutput(storage[:storage_ix>>3]) return true } { /* Theoretical max number of commands is 1 per 2 bytes. */ - var newsize uint = uint(uint32(s.num_commands_) + bytes/2 + 1) - if newsize > s.cmd_alloc_size_ { - var new_commands []command - + newsize := len(s.commands) + int(bytes)/2 + 1 + if newsize > cap(s.commands) { /* Reserve a bit more memory to allow merging with a next block without reallocation: that would impact speed. */ - newsize += uint((bytes / 4) + 16) + newsize += int(bytes/4) + 16 - s.cmd_alloc_size_ = newsize - new_commands = make([]command, newsize) - if s.commands_ != nil { - copy(new_commands, s.commands_[:s.num_commands_]) - s.commands_ = nil + new_commands := make([]command, len(s.commands), newsize) + if s.commands != nil { + copy(new_commands, s.commands) } - s.commands_ = new_commands + s.commands = new_commands } } @@ -1313,46 +830,44 @@ func encodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, outpu literal_context_mode = chooseContextMode(&s.params, data, uint(wrapPosition(s.last_flush_pos_)), uint(mask), uint(s.input_pos_-s.last_flush_pos_)) - if s.num_commands_ != 0 && s.last_insert_len_ == 0 { + if len(s.commands) != 0 && s.last_insert_len_ == 0 { extendLastCommand(s, &bytes, &wrapped_last_processed_pos) } if s.params.quality == zopflificationQuality { assert(s.params.hasher.type_ == 10) - createZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*h10), s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_) + createZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*h10), s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_) } else if s.params.quality == hqZopflificationQuality { assert(s.params.hasher.type_ == 10) - createHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_) + createHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_) } else { - createBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_) + createBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_) } { var max_length uint = maxMetablockSize(&s.params) var max_literals uint = max_length / 8 - var max_commands uint = max_length / 8 + max_commands := int(max_length / 8) var processed_bytes uint = uint(s.input_pos_ - s.last_flush_pos_) var next_input_fits_metablock bool = (processed_bytes+inputBlockSize(s) <= max_length) - var should_flush bool = (s.params.quality < minQualityForBlockSplit && s.num_literals_+s.num_commands_ >= maxNumDelayedSymbols) + var should_flush bool = (s.params.quality < minQualityForBlockSplit && s.num_literals_+uint(len(s.commands)) >= maxNumDelayedSymbols) /* If maximal possible additional block doesn't fit metablock, flush now. */ /* TODO: Postpone decision until next block arrives? */ /* If block splitting is not used, then flush as soon as there is some amount of commands / literals produced. */ - if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && s.num_commands_ < max_commands { + if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && len(s.commands) < max_commands { /* Merge with next input block. Everything will happen later. */ if updateLastProcessedPos(s) { hasherReset(s.hasher_) } - *out_size = 0 return true } } /* Create the last insert-only command. */ if s.last_insert_len_ > 0 { - initInsertCommand(&s.commands_[s.num_commands_], s.last_insert_len_) - s.num_commands_++ + s.commands = append(s.commands, makeInsertCommand(s.last_insert_len_)) s.num_literals_ += s.last_insert_len_ s.last_insert_len_ = 0 } @@ -1360,8 +875,6 @@ func encodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, outpu if !is_last && s.input_pos_ == s.last_flush_pos_ { /* We have no new input data and we don't have to finish the stream, so nothing to do. */ - *out_size = 0 - return true } @@ -1370,11 +883,11 @@ func encodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, outpu assert(s.input_pos_-s.last_flush_pos_ <= 1<<24) { var metablock_size uint32 = uint32(s.input_pos_ - s.last_flush_pos_) - var storage []byte = getBrotliStorage(s, uint(2*metablock_size+503)) + var storage []byte = s.getStorage(int(2*metablock_size + 503)) var storage_ix uint = uint(s.last_bytes_bits_) storage[0] = byte(s.last_bytes_) storage[1] = byte(s.last_bytes_ >> 8) - writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.num_commands_, s.commands_, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage) + writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.commands, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage) s.last_bytes_ = uint16(storage[storage_ix>>3]) s.last_bytes_bits_ = byte(storage_ix & 7) s.last_flush_pos_ = s.input_pos_ @@ -1390,15 +903,14 @@ func encodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, outpu s.prev_byte2_ = data[uint32(s.last_flush_pos_-2)&mask] } - s.num_commands_ = 0 + s.commands = s.commands[:0] s.num_literals_ = 0 /* Save the state of the distance cache in case we need to restore it for emitting an uncompressed block. */ copy(s.saved_dist_cache_[:], s.dist_cache_[:]) - *output = storage[0:] - *out_size = storage_ix >> 3 + s.writeOutput(storage[:storage_ix>>3]) return true } } @@ -1408,8 +920,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, outpu REQUIRED: |header| should be 8-byte aligned and at least 16 bytes long. REQUIRED: |block_size| <= (1 << 24). */ func writeMetadataHeader(s *Writer, block_size uint, header []byte) uint { - var storage_ix uint - storage_ix = uint(s.last_bytes_bits_) + storage_ix := uint(s.last_bytes_bits_) header[0] = byte(s.last_bytes_) header[1] = byte(s.last_bytes_ >> 8) s.last_bytes_ = 0 @@ -1438,7 +949,6 @@ func writeMetadataHeader(s *Writer, block_size uint, header []byte) uint { func injectBytePaddingBlock(s *Writer) { var seal uint32 = uint32(s.last_bytes_) var seal_bits uint = uint(s.last_bytes_bits_) - var destination []byte s.last_bytes_ = 0 s.last_bytes_bits_ = 0 @@ -1447,14 +957,7 @@ func injectBytePaddingBlock(s *Writer) { seal_bits += 6 - /* If we have already created storage, then append to it. - Storage is valid until next block is being compressed. */ - if s.next_out_ != nil { - destination = s.next_out_[s.available_out_:] - } else { - destination = s.tiny_buf_.u8[:] - s.next_out_ = destination - } + destination := s.tiny_buf_.u8[:] destination[0] = byte(seal) if seal_bits > 8 { @@ -1463,42 +966,35 @@ func injectBytePaddingBlock(s *Writer) { if seal_bits > 16 { destination[2] = byte(seal >> 16) } - s.available_out_ += (seal_bits + 7) >> 3 + s.writeOutput(destination[:(seal_bits+7)>>3]) } func checkFlushComplete(s *Writer) { - if s.stream_state_ == streamFlushRequested && s.available_out_ == 0 { + if s.stream_state_ == streamFlushRequested && s.err == nil { s.stream_state_ = streamProcessing - s.next_out_ = nil } } func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[]byte) bool { var block_size_limit uint = uint(1) << s.params.lgwin var buf_size uint = brotli_min_size_t(kCompressFragmentTwoPassBlockSize, brotli_min_size_t(*available_in, block_size_limit)) - var tmp_command_buf []uint32 = nil var command_buf []uint32 = nil - var tmp_literal_buf []byte = nil var literal_buf []byte = nil if s.params.quality != fastOnePassCompressionQuality && s.params.quality != fastTwoPassCompressionQuality { return false } if s.params.quality == fastTwoPassCompressionQuality { - if s.command_buf_ == nil && buf_size == kCompressFragmentTwoPassBlockSize { - s.command_buf_ = make([]uint32, kCompressFragmentTwoPassBlockSize) - s.literal_buf_ = make([]byte, kCompressFragmentTwoPassBlockSize) - } - - if s.command_buf_ != nil { - command_buf = s.command_buf_ - literal_buf = s.literal_buf_ + if s.command_buf_ == nil || cap(s.command_buf_) < int(buf_size) { + s.command_buf_ = make([]uint32, buf_size) + s.literal_buf_ = make([]byte, buf_size) } else { - tmp_command_buf = make([]uint32, buf_size) - tmp_literal_buf = make([]byte, buf_size) - command_buf = tmp_command_buf - literal_buf = tmp_literal_buf + s.command_buf_ = s.command_buf_[:buf_size] + s.literal_buf_ = s.literal_buf_[:buf_size] } + + command_buf = s.command_buf_ + literal_buf = s.literal_buf_ } for { @@ -1507,10 +1003,10 @@ func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[ continue } - /* Compress block only when internal output buffer is empty, stream is not + /* Compress block only when stream is not finished, there is no pending flush request, and there is either additional input or pending operation. */ - if s.available_out_ == 0 && s.stream_state_ == streamProcessing && (*available_in != 0 || op != int(operationProcess)) { + if s.stream_state_ == streamProcessing && (*available_in != 0 || op != int(operationProcess)) { var block_size uint = brotli_min_size_t(block_size_limit, *available_in) var is_last bool = (*available_in == block_size) && (op == int(operationFinish)) var force_flush bool = (*available_in == block_size) && (op == int(operationFlush)) @@ -1525,7 +1021,7 @@ func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[ continue } - storage = getBrotliStorage(s, max_out_size) + storage = s.getStorage(int(max_out_size)) storage[0] = byte(s.last_bytes_) storage[1] = byte(s.last_bytes_ >> 8) @@ -1540,8 +1036,7 @@ func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[ *next_in = (*next_in)[block_size:] *available_in -= block_size var out_bytes uint = storage_ix >> 3 - s.next_out_ = storage - s.available_out_ = out_bytes + s.writeOutput(storage[:out_bytes]) s.last_bytes_ = uint16(storage[storage_ix>>3]) s.last_bytes_bits_ = byte(storage_ix & 7) @@ -1558,8 +1053,6 @@ func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[ break } - tmp_command_buf = nil - tmp_literal_buf = nil checkFlushComplete(s) return true } @@ -1585,12 +1078,8 @@ func processMetadata(s *Writer, available_in *uint, next_in *[]byte) bool { continue } - if s.available_out_ != 0 { - break - } - if s.input_pos_ != s.last_flush_pos_ { - var result bool = encodeData(s, false, true, &s.available_out_, &s.next_out_) + var result bool = encodeData(s, false, true) if !result { return false } @@ -1598,8 +1087,8 @@ func processMetadata(s *Writer, available_in *uint, next_in *[]byte) bool { } if s.stream_state_ == streamMetadataHead { - s.next_out_ = s.tiny_buf_.u8[:] - s.available_out_ = writeMetadataHeader(s, uint(s.remaining_metadata_bytes_), s.next_out_) + n := writeMetadataHeader(s, uint(s.remaining_metadata_bytes_), s.tiny_buf_.u8[:]) + s.writeOutput(s.tiny_buf_.u8[:n]) s.stream_state_ = streamMetadataBody continue } else { @@ -1613,12 +1102,11 @@ func processMetadata(s *Writer, available_in *uint, next_in *[]byte) bool { /* This guarantees progress in "TakeOutput" workflow. */ var c uint32 = brotli_min_uint32_t(s.remaining_metadata_bytes_, 16) - s.next_out_ = s.tiny_buf_.u8[:] - copy(s.next_out_, (*next_in)[:c]) + copy(s.tiny_buf_.u8[:], (*next_in)[:c]) *next_in = (*next_in)[c:] *available_in -= uint(c) s.remaining_metadata_bytes_ -= c - s.available_out_ = uint(c) + s.writeOutput(s.tiny_buf_.u8[:c]) continue } @@ -1691,15 +1179,15 @@ func encoderCompressStream(s *Writer, op int, available_in *uint, next_in *[]byt continue } - /* Compress data only when internal output buffer is empty, stream is not + /* Compress data only when stream is not finished and there is no pending flush request. */ - if s.available_out_ == 0 && s.stream_state_ == streamProcessing { + if s.stream_state_ == streamProcessing { if remaining_block_size == 0 || op != int(operationProcess) { var is_last bool = ((*available_in == 0) && op == int(operationFinish)) var force_flush bool = ((*available_in == 0) && op == int(operationFlush)) var result bool updateSizeHint(s, *available_in) - result = encodeData(s, is_last, force_flush, &s.available_out_, &s.next_out_) + result = encodeData(s, is_last, force_flush) if !result { return false } @@ -1720,18 +1208,13 @@ func encoderCompressStream(s *Writer, op int, available_in *uint, next_in *[]byt return true } -func encoderHasMoreOutput(s *Writer) bool { - return s.available_out_ != 0 -} - -func encoderTakeOutput(s *Writer) []byte { - if s.available_out_ == 0 { - return nil +func (w *Writer) writeOutput(data []byte) { + if w.err != nil { + return } - result := s.next_out_[:s.available_out_] - s.total_out_ += s.available_out_ - s.available_out_ = 0 - checkFlushComplete(s) - return result + _, w.err = w.dst.Write(data) + if w.err == nil { + checkFlushComplete(w) + } } diff --git a/vendor/github.com/andybalholm/brotli/encoder.go b/vendor/github.com/andybalholm/brotli/encoder.go new file mode 100644 index 00000000..650d1e42 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/encoder.go @@ -0,0 +1,168 @@ +package brotli + +import "github.com/andybalholm/brotli/matchfinder" + +// An Encoder implements the matchfinder.Encoder interface, writing in Brotli format. +type Encoder struct { + wroteHeader bool + bw bitWriter + distCache []distanceCode +} + +func (e *Encoder) Reset() { + e.wroteHeader = false + e.bw = bitWriter{} +} + +func (e *Encoder) Encode(dst []byte, src []byte, matches []matchfinder.Match, lastBlock bool) []byte { + e.bw.dst = dst + if !e.wroteHeader { + e.bw.writeBits(4, 15) + e.wroteHeader = true + } + + var literalHisto [256]uint32 + var commandHisto [704]uint32 + var distanceHisto [64]uint32 + literalCount := 0 + commandCount := 0 + distanceCount := 0 + + if len(e.distCache) < len(matches) { + e.distCache = make([]distanceCode, len(matches)) + } + + // first pass: build the histograms + pos := 0 + + // d is the ring buffer of the last 4 distances. + d := [4]int{-10, -10, -10, -10} + for i, m := range matches { + if m.Unmatched > 0 { + for _, c := range src[pos : pos+m.Unmatched] { + literalHisto[c]++ + } + literalCount += m.Unmatched + } + + insertCode := getInsertLengthCode(uint(m.Unmatched)) + copyCode := getCopyLengthCode(uint(m.Length)) + if m.Length == 0 { + // If the stream ends with unmatched bytes, we need a dummy copy length. + copyCode = 2 + } + command := combineLengthCodes(insertCode, copyCode, false) + commandHisto[command]++ + commandCount++ + + if command >= 128 && m.Length != 0 { + var distCode distanceCode + switch m.Distance { + case d[3]: + distCode.code = 0 + case d[2]: + distCode.code = 1 + case d[1]: + distCode.code = 2 + case d[0]: + distCode.code = 3 + case d[3] - 1: + distCode.code = 4 + case d[3] + 1: + distCode.code = 5 + case d[3] - 2: + distCode.code = 6 + case d[3] + 2: + distCode.code = 7 + case d[3] - 3: + distCode.code = 8 + case d[3] + 3: + distCode.code = 9 + + // In my testing, codes 10–15 actually reduced the compression ratio. + + default: + distCode = getDistanceCode(m.Distance) + } + e.distCache[i] = distCode + distanceHisto[distCode.code]++ + distanceCount++ + if distCode.code != 0 { + d[0], d[1], d[2], d[3] = d[1], d[2], d[3], m.Distance + } + } + + pos += m.Unmatched + m.Length + } + + storeMetaBlockHeaderBW(uint(len(src)), false, &e.bw) + e.bw.writeBits(13, 0) + + var literalDepths [256]byte + var literalBits [256]uint16 + buildAndStoreHuffmanTreeFastBW(literalHisto[:], uint(literalCount), 8, literalDepths[:], literalBits[:], &e.bw) + + var commandDepths [704]byte + var commandBits [704]uint16 + buildAndStoreHuffmanTreeFastBW(commandHisto[:], uint(commandCount), 10, commandDepths[:], commandBits[:], &e.bw) + + var distanceDepths [64]byte + var distanceBits [64]uint16 + buildAndStoreHuffmanTreeFastBW(distanceHisto[:], uint(distanceCount), 6, distanceDepths[:], distanceBits[:], &e.bw) + + pos = 0 + for i, m := range matches { + insertCode := getInsertLengthCode(uint(m.Unmatched)) + copyCode := getCopyLengthCode(uint(m.Length)) + if m.Length == 0 { + // If the stream ends with unmatched bytes, we need a dummy copy length. + copyCode = 2 + } + command := combineLengthCodes(insertCode, copyCode, false) + e.bw.writeBits(uint(commandDepths[command]), uint64(commandBits[command])) + if kInsExtra[insertCode] > 0 { + e.bw.writeBits(uint(kInsExtra[insertCode]), uint64(m.Unmatched)-uint64(kInsBase[insertCode])) + } + if kCopyExtra[copyCode] > 0 { + e.bw.writeBits(uint(kCopyExtra[copyCode]), uint64(m.Length)-uint64(kCopyBase[copyCode])) + } + + if m.Unmatched > 0 { + for _, c := range src[pos : pos+m.Unmatched] { + e.bw.writeBits(uint(literalDepths[c]), uint64(literalBits[c])) + } + } + + if command >= 128 && m.Length != 0 { + distCode := e.distCache[i] + e.bw.writeBits(uint(distanceDepths[distCode.code]), uint64(distanceBits[distCode.code])) + if distCode.nExtra > 0 { + e.bw.writeBits(distCode.nExtra, distCode.extraBits) + } + } + + pos += m.Unmatched + m.Length + } + + if lastBlock { + e.bw.writeBits(2, 3) // islast + isempty + e.bw.jumpToByteBoundary() + } + return e.bw.dst +} + +type distanceCode struct { + code int + nExtra uint + extraBits uint64 +} + +func getDistanceCode(distance int) distanceCode { + d := distance + 3 + nbits := log2FloorNonZero(uint(d)) - 1 + prefix := (d >> nbits) & 1 + offset := (2 + prefix) << nbits + distcode := int(2*(nbits-1)) + prefix + 16 + extra := d - offset + return distanceCode{distcode, uint(nbits), uint64(extra)} +} diff --git a/vendor/github.com/andybalholm/brotli/entropy_encode.go b/vendor/github.com/andybalholm/brotli/entropy_encode.go index d0c1dca2..3f469a3d 100644 --- a/vendor/github.com/andybalholm/brotli/entropy_encode.go +++ b/vendor/github.com/andybalholm/brotli/entropy_encode.go @@ -24,7 +24,7 @@ func initHuffmanTree(self *huffmanTree, count uint32, left int16, right int16) { } /* Input size optimized Shell sort. */ -type huffmanTreeComparator func(*huffmanTree, *huffmanTree) bool +type huffmanTreeComparator func(huffmanTree, huffmanTree) bool var sortHuffmanTreeItems_gaps = []uint{132, 57, 23, 10, 4, 1} @@ -36,14 +36,13 @@ func sortHuffmanTreeItems(items []huffmanTree, n uint, comparator huffmanTreeCom var tmp huffmanTree = items[i] var k uint = i var j uint = i - 1 - for comparator(&tmp, &items[j]) { + for comparator(tmp, items[j]) { items[k] = items[j] k = j - tmp10 := j - j-- - if tmp10 == 0 { + if j == 0 { break } + j-- } items[k] = tmp @@ -63,7 +62,7 @@ func sortHuffmanTreeItems(items []huffmanTree, n uint, comparator huffmanTreeCom for i = gap; i < n; i++ { var j uint = i var tmp huffmanTree = items[i] - for ; j >= gap && comparator(&tmp, &items[j-gap]); j -= gap { + for ; j >= gap && comparator(tmp, items[j-gap]); j -= gap { items[j] = items[j-gap] } @@ -105,7 +104,7 @@ func setDepth(p0 int, pool []huffmanTree, depth []byte, max_depth int) bool { } /* Sort the root nodes, least popular first. */ -func sortHuffmanTree(v0 *huffmanTree, v1 *huffmanTree) bool { +func sortHuffmanTree(v0 huffmanTree, v1 huffmanTree) bool { if v0.total_count_ != v1.total_count_ { return v0.total_count_ < v1.total_count_ } diff --git a/vendor/github.com/andybalholm/brotli/entropy_encode_static.go b/vendor/github.com/andybalholm/brotli/entropy_encode_static.go index 5ddf3fcb..294aff4f 100644 --- a/vendor/github.com/andybalholm/brotli/entropy_encode_static.go +++ b/vendor/github.com/andybalholm/brotli/entropy_encode_static.go @@ -782,6 +782,11 @@ func storeStaticCodeLengthCode(storage_ix *uint, storage []byte) { writeBits(40, 0x0000FF55555554, storage_ix, storage) } +func storeStaticCodeLengthCodeBW(bw *bitWriter) { + bw.writeBits(32, 0x55555554) + bw.writeBits(8, 0xFF) +} + var kZeroRepsBits = [numCommandSymbols]uint64{ 0x00000000, 0x00000000, diff --git a/vendor/github.com/andybalholm/brotli/fast_log.go b/vendor/github.com/andybalholm/brotli/fast_log.go index bbae3009..9d6607f7 100644 --- a/vendor/github.com/andybalholm/brotli/fast_log.go +++ b/vendor/github.com/andybalholm/brotli/fast_log.go @@ -1,6 +1,9 @@ package brotli -import "math" +import ( + "math" + "math/bits" +) /* Copyright 2013 Google Inc. All Rights Reserved. @@ -11,16 +14,7 @@ import "math" /* Utilities for fast computation of logarithms. */ func log2FloorNonZero(n uint) uint32 { - /* TODO: generalize and move to platform.h */ - var result uint32 = 0 - for { - n >>= 1 - if n == 0 { - break - } - result++ - } - return result + return uint32(bits.Len(n)) - 1 } /* A lookup table for small values of log2(int) to be used in entropy diff --git a/vendor/github.com/andybalholm/brotli/find_match_length.go b/vendor/github.com/andybalholm/brotli/find_match_length.go index 14d350aa..09d2ae67 100644 --- a/vendor/github.com/andybalholm/brotli/find_match_length.go +++ b/vendor/github.com/andybalholm/brotli/find_match_length.go @@ -1,5 +1,11 @@ package brotli +import ( + "encoding/binary" + "math/bits" + "runtime" +) + /* Copyright 2010 Google Inc. All Rights Reserved. Distributed under MIT license. @@ -9,6 +15,29 @@ package brotli /* Function to find maximal matching prefixes of strings. */ func findMatchLengthWithLimit(s1 []byte, s2 []byte, limit uint) uint { var matched uint = 0 + _, _ = s1[limit-1], s2[limit-1] // bounds check + switch runtime.GOARCH { + case "amd64": + // Compare 8 bytes at at time. + for matched+8 <= limit { + w1 := binary.LittleEndian.Uint64(s1[matched:]) + w2 := binary.LittleEndian.Uint64(s2[matched:]) + if w1 != w2 { + return matched + uint(bits.TrailingZeros64(w1^w2)>>3) + } + matched += 8 + } + case "386": + // Compare 4 bytes at at time. + for matched+4 <= limit { + w1 := binary.LittleEndian.Uint32(s1[matched:]) + w2 := binary.LittleEndian.Uint32(s2[matched:]) + if w1 != w2 { + return matched + uint(bits.TrailingZeros32(w1^w2)>>3) + } + matched += 4 + } + } for matched < limit && s1[matched] == s2[matched] { matched++ } diff --git a/vendor/github.com/andybalholm/brotli/hash.go b/vendor/github.com/andybalholm/brotli/hash.go index 003b433e..00f812e8 100644 --- a/vendor/github.com/andybalholm/brotli/hash.go +++ b/vendor/github.com/andybalholm/brotli/hash.go @@ -29,8 +29,6 @@ type hasherHandle interface { Store(data []byte, mask uint, ix uint) } -type score_t uint - const kCutoffTransformsCount uint32 = 10 /* 0, 12, 27, 23, 42, 63, 56, 48, 59, 64 */ diff --git a/vendor/github.com/andybalholm/brotli/hash_forgetful_chain.go b/vendor/github.com/andybalholm/brotli/hash_forgetful_chain.go index 3364c44b..306e46d3 100644 --- a/vendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +++ b/vendor/github.com/andybalholm/brotli/hash_forgetful_chain.go @@ -110,8 +110,7 @@ func (h *hashForgetfulChain) Prepare(one_shot bool, input_size uint, data []byte func (h *hashForgetfulChain) Store(data []byte, mask uint, ix uint) { var key uint = h.HashBytes(data[ix&mask:]) var bank uint = key & (h.numBanks - 1) - var idx uint - idx = uint(h.free_slot_idx[bank]) & ((1 << h.bankBits) - 1) + idx := uint(h.free_slot_idx[bank]) & ((1 << h.bankBits) - 1) h.free_slot_idx[bank]++ var delta uint = ix - uint(h.addr[key]) h.tiny_hash[uint16(ix)] = byte(key) diff --git a/vendor/github.com/andybalholm/brotli/hash_rolling.go b/vendor/github.com/andybalholm/brotli/hash_rolling.go index ad655a0a..6630fc07 100644 --- a/vendor/github.com/andybalholm/brotli/hash_rolling.go +++ b/vendor/github.com/andybalholm/brotli/hash_rolling.go @@ -48,7 +48,6 @@ type hashRolling struct { state uint32 table []uint32 next_ix uint - chunk_len uint32 factor uint32 factor_remove uint32 } diff --git a/vendor/github.com/andybalholm/brotli/histogram.go b/vendor/github.com/andybalholm/brotli/histogram.go index f208ff74..0346622b 100644 --- a/vendor/github.com/andybalholm/brotli/histogram.go +++ b/vendor/github.com/andybalholm/brotli/histogram.go @@ -163,7 +163,7 @@ func initBlockSplitIterator(self *blockSplitIterator, split *blockSplit) { self.split_ = split self.idx_ = 0 self.type_ = 0 - if split.lengths != nil { + if len(split.lengths) > 0 { self.length_ = uint(split.lengths[0]) } else { self.length_ = 0 @@ -180,17 +180,16 @@ func blockSplitIteratorNext(self *blockSplitIterator) { self.length_-- } -func buildHistogramsWithContext(cmds []command, num_commands uint, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) { +func buildHistogramsWithContext(cmds []command, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) { var pos uint = start_pos var literal_it blockSplitIterator var insert_and_copy_it blockSplitIterator var dist_it blockSplitIterator - var i uint initBlockSplitIterator(&literal_it, literal_split) initBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split) initBlockSplitIterator(&dist_it, dist_split) - for i = 0; i < num_commands; i++ { + for i := range cmds { var cmd *command = &cmds[i] var j uint blockSplitIteratorNext(&insert_and_copy_it) diff --git a/vendor/github.com/andybalholm/brotli/http.go b/vendor/github.com/andybalholm/brotli/http.go new file mode 100644 index 00000000..3d3a8a06 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/http.go @@ -0,0 +1,184 @@ +package brotli + +import ( + "compress/gzip" + "io" + "net/http" + "strings" +) + +// HTTPCompressor chooses a compression method (brotli, gzip, or none) based on +// the Accept-Encoding header, sets the Content-Encoding header, and returns a +// WriteCloser that implements that compression. The Close method must be called +// before the current HTTP handler returns. +func HTTPCompressor(w http.ResponseWriter, r *http.Request) io.WriteCloser { + if w.Header().Get("Vary") == "" { + w.Header().Set("Vary", "Accept-Encoding") + } + + encoding := negotiateContentEncoding(r, []string{"br", "gzip"}) + switch encoding { + case "br": + w.Header().Set("Content-Encoding", "br") + return NewWriterV2(w, DefaultCompression) + case "gzip": + w.Header().Set("Content-Encoding", "gzip") + return gzip.NewWriter(w) + } + return nopCloser{w} +} + +// negotiateContentEncoding returns the best offered content encoding for the +// request's Accept-Encoding header. If two offers match with equal weight and +// then the offer earlier in the list is preferred. If no offers are +// acceptable, then "" is returned. +func negotiateContentEncoding(r *http.Request, offers []string) string { + bestOffer := "identity" + bestQ := -1.0 + specs := parseAccept(r.Header, "Accept-Encoding") + for _, offer := range offers { + for _, spec := range specs { + if spec.Q > bestQ && + (spec.Value == "*" || spec.Value == offer) { + bestQ = spec.Q + bestOffer = offer + } + } + } + if bestQ == 0 { + bestOffer = "" + } + return bestOffer +} + +// acceptSpec describes an Accept* header. +type acceptSpec struct { + Value string + Q float64 +} + +// parseAccept parses Accept* headers. +func parseAccept(header http.Header, key string) (specs []acceptSpec) { +loop: + for _, s := range header[key] { + for { + var spec acceptSpec + spec.Value, s = expectTokenSlash(s) + if spec.Value == "" { + continue loop + } + spec.Q = 1.0 + s = skipSpace(s) + if strings.HasPrefix(s, ";") { + s = skipSpace(s[1:]) + if !strings.HasPrefix(s, "q=") { + continue loop + } + spec.Q, s = expectQuality(s[2:]) + if spec.Q < 0.0 { + continue loop + } + } + specs = append(specs, spec) + s = skipSpace(s) + if !strings.HasPrefix(s, ",") { + continue loop + } + s = skipSpace(s[1:]) + } + } + return +} + +func skipSpace(s string) (rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isSpace == 0 { + break + } + } + return s[i:] +} + +func expectTokenSlash(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + b := s[i] + if (octetTypes[b]&isToken == 0) && b != '/' { + break + } + } + return s[:i], s[i:] +} + +func expectQuality(s string) (q float64, rest string) { + switch { + case len(s) == 0: + return -1, "" + case s[0] == '0': + q = 0 + case s[0] == '1': + q = 1 + default: + return -1, "" + } + s = s[1:] + if !strings.HasPrefix(s, ".") { + return q, s + } + s = s[1:] + i := 0 + n := 0 + d := 1 + for ; i < len(s); i++ { + b := s[i] + if b < '0' || b > '9' { + break + } + n = n*10 + int(b) - '0' + d *= 10 + } + return q + float64(n)/float64(d), s[i:] +} + +// Octet types from RFC 2616. +var octetTypes [256]octetType + +type octetType byte + +const ( + isToken octetType = 1 << iota + isSpace +) + +func init() { + // OCTET = + // CHAR = + // CTL = + // CR = + // LF = + // SP = + // HT = + // <"> = + // CRLF = CR LF + // LWS = [CRLF] 1*( SP | HT ) + // TEXT = + // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> + // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT + // token = 1* + // qdtext = > + + for c := 0; c < 256; c++ { + var t octetType + isCtl := c <= 31 || c == 127 + isChar := 0 <= c && c <= 127 + isSeparator := strings.ContainsRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) + if strings.ContainsRune(" \t\r\n", rune(c)) { + t |= isSpace + } + if isChar && !isCtl && !isSeparator { + t |= isToken + } + octetTypes[c] = t + } +} diff --git a/vendor/github.com/andybalholm/brotli/matchfinder/emitter.go b/vendor/github.com/andybalholm/brotli/matchfinder/emitter.go new file mode 100644 index 00000000..37ed8e13 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/matchfinder/emitter.go @@ -0,0 +1,45 @@ +package matchfinder + +// An absoluteMatch is like a Match, but it stores indexes into the byte +// stream instead of lengths. +type absoluteMatch struct { + // Start is the index of the first byte. + Start int + + // End is the index of the byte after the last byte + // (so that End - Start = Length). + End int + + // Match is the index of the previous data that matches + // (Start - Match = Distance). + Match int +} + +// A matchEmitter manages the output of matches for a MatchFinder. +type matchEmitter struct { + // Dst is the destination slice that Matches are added to. + Dst []Match + + // NextEmit is the index of the next byte to emit. + NextEmit int +} + +func (e *matchEmitter) emit(m absoluteMatch) { + e.Dst = append(e.Dst, Match{ + Unmatched: m.Start - e.NextEmit, + Length: m.End - m.Start, + Distance: m.Start - m.Match, + }) + e.NextEmit = m.End +} + +// trim shortens m if it extends past maxEnd. Then if the length is at least +// minLength, the match is emitted. +func (e *matchEmitter) trim(m absoluteMatch, maxEnd int, minLength int) { + if m.End > maxEnd { + m.End = maxEnd + } + if m.End-m.Start >= minLength { + e.emit(m) + } +} diff --git a/vendor/github.com/andybalholm/brotli/matchfinder/m0.go b/vendor/github.com/andybalholm/brotli/matchfinder/m0.go new file mode 100644 index 00000000..773b7c49 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/matchfinder/m0.go @@ -0,0 +1,169 @@ +package matchfinder + +import ( + "encoding/binary" +) + +// M0 is an implementation of the MatchFinder interface based +// on the algorithm used by snappy, but modified to be more like the algorithm +// used by compression level 0 of the brotli reference implementation. +// +// It has a maximum block size of 65536 bytes. +type M0 struct { + // Lazy turns on "lazy matching," for higher compression but less speed. + Lazy bool + + MaxDistance int + MaxLength int +} + +func (M0) Reset() {} + +const ( + m0HashLen = 5 + + m0TableBits = 14 + m0TableSize = 1 << m0TableBits + m0Shift = 32 - m0TableBits + // m0TableMask is redundant, but helps the compiler eliminate bounds + // checks. + m0TableMask = m0TableSize - 1 +) + +func (m M0) hash(data uint64) uint64 { + hash := (data << (64 - 8*m0HashLen)) * hashMul64 + return hash >> (64 - m0TableBits) +} + +// FindMatches looks for matches in src, appends them to dst, and returns dst. +// src must not be longer than 65536 bytes. +func (m M0) FindMatches(dst []Match, src []byte) []Match { + const inputMargin = 16 - 1 + const minNonLiteralBlockSize = 1 + 1 + inputMargin + + if len(src) < minNonLiteralBlockSize { + dst = append(dst, Match{ + Unmatched: len(src), + }) + return dst + } + if len(src) > 65536 { + panic("block too long") + } + + var table [m0TableSize]uint16 + + // sLimit is when to stop looking for offset/length copies. The inputMargin + // lets us use a fast path for emitLiteral in the main loop, while we are + // looking for copies. + sLimit := len(src) - inputMargin + + // nextEmit is where in src the next emitLiteral should start from. + nextEmit := 0 + + // The encoded form must start with a literal, as there are no previous + // bytes to copy, so we start looking for hash matches at s == 1. + s := 1 + nextHash := m.hash(binary.LittleEndian.Uint64(src[s:])) + + for { + // Copied from the C++ snappy implementation: + // + // Heuristic match skipping: If 32 bytes are scanned with no matches + // found, start looking only at every other byte. If 32 more bytes are + // scanned (or skipped), look at every third byte, etc.. When a match + // is found, immediately go back to looking at every byte. This is a + // small loss (~5% performance, ~0.1% density) for compressible data + // due to more bookkeeping, but for non-compressible data (such as + // JPEG) it's a huge win since the compressor quickly "realizes" the + // data is incompressible and doesn't bother looking for matches + // everywhere. + // + // The "skip" variable keeps track of how many bytes there are since + // the last match; dividing it by 32 (ie. right-shifting by five) gives + // the number of bytes to move ahead for each iteration. + skip := 32 + + nextS := s + candidate := 0 + for { + s = nextS + bytesBetweenHashLookups := skip >> 5 + nextS = s + bytesBetweenHashLookups + skip += bytesBetweenHashLookups + if nextS > sLimit { + goto emitRemainder + } + candidate = int(table[nextHash&m0TableMask]) + table[nextHash&m0TableMask] = uint16(s) + nextHash = m.hash(binary.LittleEndian.Uint64(src[nextS:])) + if m.MaxDistance != 0 && s-candidate > m.MaxDistance { + continue + } + if binary.LittleEndian.Uint32(src[s:]) == binary.LittleEndian.Uint32(src[candidate:]) { + break + } + } + + // Invariant: we have a 4-byte match at s. + base := s + s = extendMatch(src, candidate+4, s+4) + + origBase := base + if m.Lazy && base+1 < sLimit { + newBase := base + 1 + h := m.hash(binary.LittleEndian.Uint64(src[newBase:])) + newCandidate := int(table[h&m0TableMask]) + table[h&m0TableMask] = uint16(newBase) + okDistance := true + if m.MaxDistance != 0 && newBase-newCandidate > m.MaxDistance { + okDistance = false + } + if okDistance && binary.LittleEndian.Uint32(src[newBase:]) == binary.LittleEndian.Uint32(src[newCandidate:]) { + newS := extendMatch(src, newCandidate+4, newBase+4) + if newS-newBase > s-base+1 { + s = newS + base = newBase + candidate = newCandidate + } + } + } + + if m.MaxLength != 0 && s-base > m.MaxLength { + s = base + m.MaxLength + } + dst = append(dst, Match{ + Unmatched: base - nextEmit, + Length: s - base, + Distance: base - candidate, + }) + nextEmit = s + if s >= sLimit { + goto emitRemainder + } + + if m.Lazy { + // If lazy matching is enabled, we update the hash table for + // every byte in the match. + for i := origBase + 2; i < s-1; i++ { + x := binary.LittleEndian.Uint64(src[i:]) + table[m.hash(x)&m0TableMask] = uint16(i) + } + } + + // We could immediately start working at s now, but to improve + // compression we first update the hash table at s-1 and at s. + x := binary.LittleEndian.Uint64(src[s-1:]) + prevHash := m.hash(x >> 0) + table[prevHash&m0TableMask] = uint16(s - 1) + nextHash = m.hash(x >> 8) + } + +emitRemainder: + if nextEmit < len(src) { + dst = append(dst, Match{ + Unmatched: len(src) - nextEmit, + }) + } + return dst +} diff --git a/vendor/github.com/andybalholm/brotli/matchfinder/m4.go b/vendor/github.com/andybalholm/brotli/matchfinder/m4.go new file mode 100644 index 00000000..5b2acba2 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/matchfinder/m4.go @@ -0,0 +1,297 @@ +package matchfinder + +import ( + "encoding/binary" + "math/bits" + "runtime" +) + +// M4 is an implementation of the MatchFinder +// interface that uses a hash table to find matches, +// optional match chains, +// and the advanced parsing technique from +// https://fastcompression.blogspot.com/2011/12/advanced-parsing-strategies.html. +type M4 struct { + // MaxDistance is the maximum distance (in bytes) to look back for + // a match. The default is 65535. + MaxDistance int + + // MinLength is the length of the shortest match to return. + // The default is 4. + MinLength int + + // HashLen is the number of bytes to use to calculate the hashes. + // The maximum is 8 and the default is 6. + HashLen int + + // TableBits is the number of bits in the hash table indexes. + // The default is 17 (128K entries). + TableBits int + + // ChainLength is how many entries to search on the "match chain" of older + // locations with the same hash as the current location. + ChainLength int + + // DistanceBitCost is used when comparing two matches to see + // which is better. The comparison is primarily based on the length + // of the matches, but it can also take the distance into account, + // in terms of the number of bits needed to represent the distance. + // One byte of length is given a score of 256, so 32 (256/8) would + // be a reasonable first guess for the value of one bit. + // (The default is 0, which bases the comparison solely on length.) + DistanceBitCost int + + table []uint32 + chain []uint16 + + history []byte +} + +func (q *M4) Reset() { + for i := range q.table { + q.table[i] = 0 + } + q.history = q.history[:0] + q.chain = q.chain[:0] +} + +func (q *M4) score(m absoluteMatch) int { + return (m.End-m.Start)*256 + bits.LeadingZeros32(uint32(m.Start-m.Match))*q.DistanceBitCost +} + +func (q *M4) FindMatches(dst []Match, src []byte) []Match { + if q.MaxDistance == 0 { + q.MaxDistance = 65535 + } + if q.MinLength == 0 { + q.MinLength = 4 + } + if q.HashLen == 0 { + q.HashLen = 6 + } + if q.TableBits == 0 { + q.TableBits = 17 + } + if len(q.table) < 1< q.MaxDistance*2 { + // Trim down the history buffer. + delta := len(q.history) - q.MaxDistance + copy(q.history, q.history[delta:]) + q.history = q.history[:q.MaxDistance] + if q.ChainLength > 0 { + q.chain = q.chain[:q.MaxDistance] + } + + for i, v := range q.table { + newV := int(v) - delta + if newV < 0 { + newV = 0 + } + q.table[i] = uint32(newV) + } + } + + // Append src to the history buffer. + e.NextEmit = len(q.history) + q.history = append(q.history, src...) + if q.ChainLength > 0 { + q.chain = append(q.chain, make([]uint16, len(src))...) + } + src = q.history + + // matches stores the matches that have been found but not emitted, + // in reverse order. (matches[0] is the most recent one.) + var matches [3]absoluteMatch + for i := e.NextEmit; i < len(src)-7; i++ { + if matches[0] != (absoluteMatch{}) && i >= matches[0].End { + // We have found some matches, and we're far enough along that we probably + // won't find overlapping matches, so we might as well emit them. + if matches[1] != (absoluteMatch{}) { + e.trim(matches[1], matches[0].Start, q.MinLength) + } + e.emit(matches[0]) + matches = [3]absoluteMatch{} + } + + // Calculate and store the hash. + h := ((binary.LittleEndian.Uint64(src[i:]) & (1<<(8*q.HashLen) - 1)) * hashMul64) >> (64 - q.TableBits) + candidate := int(q.table[h]) + q.table[h] = uint32(i) + if q.ChainLength > 0 && candidate != 0 { + delta := i - candidate + if delta < 1<<16 { + q.chain[i] = uint16(delta) + } + } + + if i < matches[0].End && i != matches[0].End+2-q.HashLen { + continue + } + if candidate == 0 || i-candidate > q.MaxDistance { + continue + } + + // Look for a match. + var currentMatch absoluteMatch + + if i-candidate != matches[0].Start-matches[0].Match { + if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) { + m := extendMatch2(src, i, candidate, e.NextEmit) + if m.End-m.Start > q.MinLength { + currentMatch = m + } + } + } + + for j := 0; j < q.ChainLength; j++ { + delta := q.chain[candidate] + if delta == 0 { + break + } + candidate -= int(delta) + if candidate <= 0 || i-candidate > q.MaxDistance { + break + } + if i-candidate != matches[0].Start-matches[0].Match { + if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) { + m := extendMatch2(src, i, candidate, e.NextEmit) + if m.End-m.Start > q.MinLength && q.score(m) > q.score(currentMatch) { + currentMatch = m + } + } + } + } + + if currentMatch.End-currentMatch.Start < q.MinLength { + continue + } + + overlapPenalty := 0 + if matches[0] != (absoluteMatch{}) { + overlapPenalty = 275 + if currentMatch.Start <= matches[1].End { + // This match would completely replace the previous match, + // so there is no penalty for overlap. + overlapPenalty = 0 + } + } + + if q.score(currentMatch) <= q.score(matches[0])+overlapPenalty { + continue + } + + matches = [3]absoluteMatch{ + currentMatch, + matches[0], + matches[1], + } + + if matches[2] == (absoluteMatch{}) { + continue + } + + // We have three matches, so it's time to emit one and/or eliminate one. + switch { + case matches[0].Start < matches[2].End: + // The first and third matches overlap; discard the one in between. + matches = [3]absoluteMatch{ + matches[0], + matches[2], + absoluteMatch{}, + } + + case matches[0].Start < matches[2].End+q.MinLength: + // The first and third matches don't overlap, but there's no room for + // another match between them. Emit the first match and discard the second. + e.emit(matches[2]) + matches = [3]absoluteMatch{ + matches[0], + absoluteMatch{}, + absoluteMatch{}, + } + + default: + // Emit the first match, shortening it if necessary to avoid overlap with the second. + e.trim(matches[2], matches[1].Start, q.MinLength) + matches[2] = absoluteMatch{} + } + } + + // We've found all the matches now; emit the remaining ones. + if matches[1] != (absoluteMatch{}) { + e.trim(matches[1], matches[0].Start, q.MinLength) + } + if matches[0] != (absoluteMatch{}) { + e.emit(matches[0]) + } + + dst = e.Dst + if e.NextEmit < len(src) { + dst = append(dst, Match{ + Unmatched: len(src) - e.NextEmit, + }) + } + + return dst +} + +const hashMul64 = 0x1E35A7BD1E35A7BD + +// extendMatch returns the largest k such that k <= len(src) and that +// src[i:i+k-j] and src[j:k] have the same contents. +// +// It assumes that: +// +// 0 <= i && i < j && j <= len(src) +func extendMatch(src []byte, i, j int) int { + switch runtime.GOARCH { + case "amd64": + // As long as we are 8 or more bytes before the end of src, we can load and + // compare 8 bytes at a time. If those 8 bytes are equal, repeat. + for j+8 < len(src) { + iBytes := binary.LittleEndian.Uint64(src[i:]) + jBytes := binary.LittleEndian.Uint64(src[j:]) + if iBytes != jBytes { + // If those 8 bytes were not equal, XOR the two 8 byte values, and return + // the index of the first byte that differs. The BSF instruction finds the + // least significant 1 bit, the amd64 architecture is little-endian, and + // the shift by 3 converts a bit index to a byte index. + return j + bits.TrailingZeros64(iBytes^jBytes)>>3 + } + i, j = i+8, j+8 + } + case "386": + // On a 32-bit CPU, we do it 4 bytes at a time. + for j+4 < len(src) { + iBytes := binary.LittleEndian.Uint32(src[i:]) + jBytes := binary.LittleEndian.Uint32(src[j:]) + if iBytes != jBytes { + return j + bits.TrailingZeros32(iBytes^jBytes)>>3 + } + i, j = i+4, j+4 + } + } + for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 { + } + return j +} + +// Given a 4-byte match at src[start] and src[candidate], extendMatch2 extends it +// upward as far as possible, and downward no farther than to min. +func extendMatch2(src []byte, start, candidate, min int) absoluteMatch { + end := extendMatch(src, candidate+4, start+4) + for start > min && candidate > 0 && src[start-1] == src[candidate-1] { + start-- + candidate-- + } + return absoluteMatch{ + Start: start, + End: end, + Match: candidate, + } +} diff --git a/vendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go b/vendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go new file mode 100644 index 00000000..f6bcfdb3 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go @@ -0,0 +1,103 @@ +// The matchfinder package defines reusable components for data compression. +// +// Many compression libraries have two main parts: +// - Something that looks for repeated sequences of bytes +// - An encoder for the compressed data format (often an entropy coder) +// +// Although these are logically two separate steps, the implementations are +// usually closely tied together. You can't use flate's matcher with snappy's +// encoder, for example. This package defines interfaces and an intermediate +// representation to allow mixing and matching compression components. +package matchfinder + +import "io" + +// A Match is the basic unit of LZ77 compression. +type Match struct { + Unmatched int // the number of unmatched bytes since the previous match + Length int // the number of bytes in the matched string; it may be 0 at the end of the input + Distance int // how far back in the stream to copy from +} + +// A MatchFinder performs the LZ77 stage of compression, looking for matches. +type MatchFinder interface { + // FindMatches looks for matches in src, appends them to dst, and returns dst. + FindMatches(dst []Match, src []byte) []Match + + // Reset clears any internal state, preparing the MatchFinder to be used with + // a new stream. + Reset() +} + +// An Encoder encodes the data in its final format. +type Encoder interface { + // Encode appends the encoded format of src to dst, using the match + // information from matches. + Encode(dst []byte, src []byte, matches []Match, lastBlock bool) []byte + + // Reset clears any internal state, preparing the Encoder to be used with + // a new stream. + Reset() +} + +// A Writer uses MatchFinder and Encoder to write compressed data to Dest. +type Writer struct { + Dest io.Writer + MatchFinder MatchFinder + Encoder Encoder + + // BlockSize is the number of bytes to compress at a time. If it is zero, + // each Write operation will be treated as one block. + BlockSize int + + err error + inBuf []byte + outBuf []byte + matches []Match +} + +func (w *Writer) Write(p []byte) (n int, err error) { + if w.err != nil { + return 0, w.err + } + + if w.BlockSize == 0 { + return w.writeBlock(p, false) + } + + w.inBuf = append(w.inBuf, p...) + var pos int + for pos = 0; pos+w.BlockSize <= len(w.inBuf) && w.err == nil; pos += w.BlockSize { + w.writeBlock(w.inBuf[pos:pos+w.BlockSize], false) + } + if pos > 0 { + n := copy(w.inBuf, w.inBuf[pos:]) + w.inBuf = w.inBuf[:n] + } + + return len(p), w.err +} + +func (w *Writer) writeBlock(p []byte, lastBlock bool) (n int, err error) { + w.outBuf = w.outBuf[:0] + w.matches = w.MatchFinder.FindMatches(w.matches[:0], p) + w.outBuf = w.Encoder.Encode(w.outBuf, p, w.matches, lastBlock) + _, w.err = w.Dest.Write(w.outBuf) + return len(p), w.err +} + +func (w *Writer) Close() error { + w.writeBlock(w.inBuf, true) + w.inBuf = w.inBuf[:0] + return w.err +} + +func (w *Writer) Reset(newDest io.Writer) { + w.MatchFinder.Reset() + w.Encoder.Reset() + w.err = nil + w.inBuf = w.inBuf[:0] + w.outBuf = w.outBuf[:0] + w.matches = w.matches[:0] + w.Dest = newDest +} diff --git a/vendor/github.com/andybalholm/brotli/matchfinder/textencoder.go b/vendor/github.com/andybalholm/brotli/matchfinder/textencoder.go new file mode 100644 index 00000000..75ecc590 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/matchfinder/textencoder.go @@ -0,0 +1,53 @@ +package matchfinder + +import "fmt" + +// A TextEncoder is an Encoder that produces a human-readable representation of +// the LZ77 compression. Matches are replaced with symbols. +type TextEncoder struct{} + +func (t TextEncoder) Reset() {} + +func (t TextEncoder) Encode(dst []byte, src []byte, matches []Match, lastBlock bool) []byte { + pos := 0 + for _, m := range matches { + if m.Unmatched > 0 { + dst = append(dst, src[pos:pos+m.Unmatched]...) + pos += m.Unmatched + } + if m.Length > 0 { + dst = append(dst, []byte(fmt.Sprintf("<%d,%d>", m.Length, m.Distance))...) + pos += m.Length + } + } + if pos < len(src) { + dst = append(dst, src[pos:]...) + } + return dst +} + +// A NoMatchFinder implements MatchFinder, but doesn't find any matches. +// It can be used to implement the equivalent of the standard library flate package's +// HuffmanOnly setting. +type NoMatchFinder struct{} + +func (n NoMatchFinder) Reset() {} + +func (n NoMatchFinder) FindMatches(dst []Match, src []byte) []Match { + return append(dst, Match{ + Unmatched: len(src), + }) +} + +// AutoReset wraps a MatchFinder that can return references to data in previous +// blocks, and calls Reset before each block. It is useful for (e.g.) using a +// snappy Encoder with a MatchFinder designed for flate. (Snappy doesn't +// support references between blocks.) +type AutoReset struct { + MatchFinder +} + +func (a AutoReset) FindMatches(dst []Match, src []byte) []Match { + a.Reset() + return a.MatchFinder.FindMatches(dst, src) +} diff --git a/vendor/github.com/andybalholm/brotli/memory.go b/vendor/github.com/andybalholm/brotli/memory.go index 7208a3bb..a07c7050 100644 --- a/vendor/github.com/andybalholm/brotli/memory.go +++ b/vendor/github.com/andybalholm/brotli/memory.go @@ -23,12 +23,18 @@ func brotli_ensure_capacity_uint8_t(a *[]byte, c *uint, r uint) { for new_size < r { new_size *= 2 } - var new_array []byte = make([]byte, new_size) - if *c != 0 { - copy(new_array, (*a)[:*c]) + + if cap(*a) < int(new_size) { + var new_array []byte = make([]byte, new_size) + if *c != 0 { + copy(new_array, (*a)[:*c]) + } + + *a = new_array + } else { + *a = (*a)[:new_size] } - *a = new_array *c = new_size } } @@ -45,12 +51,16 @@ func brotli_ensure_capacity_uint32_t(a *[]uint32, c *uint, r uint) { new_size *= 2 } - new_array = make([]uint32, new_size) - if *c != 0 { - copy(new_array, (*a)[:*c]) - } + if cap(*a) < int(new_size) { + new_array = make([]uint32, new_size) + if *c != 0 { + copy(new_array, (*a)[:*c]) + } - *a = new_array + *a = new_array + } else { + *a = (*a)[:new_size] + } *c = new_size } } diff --git a/vendor/github.com/andybalholm/brotli/metablock.go b/vendor/github.com/andybalholm/brotli/metablock.go index 4a412cf4..3014df8c 100644 --- a/vendor/github.com/andybalholm/brotli/metablock.go +++ b/vendor/github.com/andybalholm/brotli/metablock.go @@ -1,5 +1,9 @@ package brotli +import ( + "sync" +) + /* Copyright 2014 Google Inc. All Rights Reserved. Distributed under MIT license. @@ -25,31 +29,30 @@ type metaBlockSplit struct { distance_histograms_size uint } -func initMetaBlockSplit(mb *metaBlockSplit) { - initBlockSplit(&mb.literal_split) - initBlockSplit(&mb.command_split) - initBlockSplit(&mb.distance_split) - mb.literal_context_map = nil - mb.literal_context_map_size = 0 - mb.distance_context_map = nil - mb.distance_context_map_size = 0 - mb.literal_histograms = nil - mb.literal_histograms_size = 0 - mb.command_histograms = nil - mb.command_histograms_size = 0 - mb.distance_histograms = nil - mb.distance_histograms_size = 0 +var metaBlockPool sync.Pool + +func getMetaBlockSplit() *metaBlockSplit { + mb, _ := metaBlockPool.Get().(*metaBlockSplit) + + if mb == nil { + mb = &metaBlockSplit{} + } else { + initBlockSplit(&mb.literal_split) + initBlockSplit(&mb.command_split) + initBlockSplit(&mb.distance_split) + mb.literal_context_map = mb.literal_context_map[:0] + mb.literal_context_map_size = 0 + mb.distance_context_map = mb.distance_context_map[:0] + mb.distance_context_map_size = 0 + mb.literal_histograms = mb.literal_histograms[:0] + mb.command_histograms = mb.command_histograms[:0] + mb.distance_histograms = mb.distance_histograms[:0] + } + return mb } -func destroyMetaBlockSplit(mb *metaBlockSplit) { - destroyBlockSplit(&mb.literal_split) - destroyBlockSplit(&mb.command_split) - destroyBlockSplit(&mb.distance_split) - mb.literal_context_map = nil - mb.distance_context_map = nil - mb.literal_histograms = nil - mb.command_histograms = nil - mb.distance_histograms = nil +func freeMetaBlockSplit(mb *metaBlockSplit) { + metaBlockPool.Put(mb) } func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32) { @@ -84,14 +87,12 @@ func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32) dist_params.max_distance = uint(max_distance) } -func recomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams) { - var i uint - +func recomputeDistancePrefixes(cmds []command, orig_params *distanceParams, new_params *distanceParams) { if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes { return } - for i = 0; i < num_commands; i++ { + for i := range cmds { var cmd *command = &cmds[i] if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { prefixEncodeCopyDistance(uint(commandRestoreDistanceCode(cmd, orig_params)), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_) @@ -99,8 +100,7 @@ func recomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *d } } -func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool { - var i uint +func computeDistanceCost(cmds []command, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool { var equal_params bool = false var dist_prefix uint16 var dist_extra uint32 @@ -112,8 +112,8 @@ func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanc equal_params = true } - for i = 0; i < num_commands; i++ { - var cmd *command = &cmds[i] + for i := range cmds { + cmd := &cmds[i] if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { if equal_params { dist_prefix = cmd.dist_prefix_ @@ -137,7 +137,7 @@ func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanc var buildMetaBlock_kMaxNumberOfHistograms uint = 256 -func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, num_commands uint, literal_context_mode int, mb *metaBlockSplit) { +func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, literal_context_mode int, mb *metaBlockSplit) { var distance_histograms []histogramDistance var literal_histograms []histogramLiteral var literal_context_modes []int = nil @@ -164,7 +164,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam check_orig = false } - skip = !computeDistanceCost(cmds, num_commands, &orig_params.dist, &new_params.dist, &dist_cost) + skip = !computeDistanceCost(cmds, &orig_params.dist, &new_params.dist, &dist_cost) if skip || (dist_cost > best_dist_cost) { break } @@ -181,7 +181,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam if check_orig { var dist_cost float64 - computeDistanceCost(cmds, num_commands, &orig_params.dist, &orig_params.dist, &dist_cost) + computeDistanceCost(cmds, &orig_params.dist, &orig_params.dist, &dist_cost) if dist_cost < best_dist_cost { /* NB: currently unused; uncomment when more param tuning is added. */ /* best_dist_cost = dist_cost; */ @@ -189,9 +189,9 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam } } - recomputeDistancePrefixes(cmds, num_commands, &orig_params.dist, ¶ms.dist) + recomputeDistancePrefixes(cmds, &orig_params.dist, ¶ms.dist) - splitBlock(cmds, num_commands, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split) + splitBlock(cmds, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split) if !params.disable_literal_context_modeling { literal_context_multiplier = 1 << literalContextBits @@ -209,21 +209,30 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam distance_histograms = make([]histogramDistance, distance_histograms_size) clearHistogramsDistance(distance_histograms, distance_histograms_size) - assert(mb.command_histograms == nil) mb.command_histograms_size = mb.command_split.num_types - mb.command_histograms = make([]histogramCommand, (mb.command_histograms_size)) + if cap(mb.command_histograms) < int(mb.command_histograms_size) { + mb.command_histograms = make([]histogramCommand, (mb.command_histograms_size)) + } else { + mb.command_histograms = mb.command_histograms[:mb.command_histograms_size] + } clearHistogramsCommand(mb.command_histograms, mb.command_histograms_size) - buildHistogramsWithContext(cmds, num_commands, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms) + buildHistogramsWithContext(cmds, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms) literal_context_modes = nil - assert(mb.literal_context_map == nil) mb.literal_context_map_size = mb.literal_split.num_types << literalContextBits - mb.literal_context_map = make([]uint32, (mb.literal_context_map_size)) + if cap(mb.literal_context_map) < int(mb.literal_context_map_size) { + mb.literal_context_map = make([]uint32, (mb.literal_context_map_size)) + } else { + mb.literal_context_map = mb.literal_context_map[:mb.literal_context_map_size] + } - assert(mb.literal_histograms == nil) mb.literal_histograms_size = mb.literal_context_map_size - mb.literal_histograms = make([]histogramLiteral, (mb.literal_histograms_size)) + if cap(mb.literal_histograms) < int(mb.literal_histograms_size) { + mb.literal_histograms = make([]histogramLiteral, (mb.literal_histograms_size)) + } else { + mb.literal_histograms = mb.literal_histograms[:mb.literal_histograms_size] + } clusterHistogramsLiteral(literal_histograms, literal_histograms_size, buildMetaBlock_kMaxNumberOfHistograms, mb.literal_histograms, &mb.literal_histograms_size, mb.literal_context_map) literal_histograms = nil @@ -239,13 +248,19 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam } } - assert(mb.distance_context_map == nil) mb.distance_context_map_size = mb.distance_split.num_types << distanceContextBits - mb.distance_context_map = make([]uint32, (mb.distance_context_map_size)) + if cap(mb.distance_context_map) < int(mb.distance_context_map_size) { + mb.distance_context_map = make([]uint32, (mb.distance_context_map_size)) + } else { + mb.distance_context_map = mb.distance_context_map[:mb.distance_context_map_size] + } - assert(mb.distance_histograms == nil) mb.distance_histograms_size = mb.distance_context_map_size - mb.distance_histograms = make([]histogramDistance, (mb.distance_histograms_size)) + if cap(mb.distance_histograms) < int(mb.distance_histograms_size) { + mb.distance_histograms = make([]histogramDistance, (mb.distance_histograms_size)) + } else { + mb.distance_histograms = mb.distance_histograms[:mb.distance_histograms_size] + } clusterHistogramsDistance(distance_histograms, mb.distance_context_map_size, buildMetaBlock_kMaxNumberOfHistograms, mb.distance_histograms, &mb.distance_histograms_size, mb.distance_context_map) distance_histograms = nil @@ -298,9 +313,12 @@ func initContextBlockSplitter(self *contextBlockSplitter, alphabet_size uint, nu brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks) brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks) split.num_blocks = max_num_blocks - assert(*histograms == nil) *histograms_size = max_num_types * num_contexts - *histograms = make([]histogramLiteral, (*histograms_size)) + if histograms == nil || cap(*histograms) < int(*histograms_size) { + *histograms = make([]histogramLiteral, (*histograms_size)) + } else { + *histograms = (*histograms)[:*histograms_size] + } self.histograms_ = *histograms /* Clear only current histogram. */ @@ -453,9 +471,12 @@ func contextBlockSplitterAddSymbol(self *contextBlockSplitter, symbol uint, cont func mapStaticContexts(num_contexts uint, static_context_map []uint32, mb *metaBlockSplit) { var i uint - assert(mb.literal_context_map == nil) mb.literal_context_map_size = mb.literal_split.num_types << literalContextBits - mb.literal_context_map = make([]uint32, (mb.literal_context_map_size)) + if cap(mb.literal_context_map) < int(mb.literal_context_map_size) { + mb.literal_context_map = make([]uint32, (mb.literal_context_map_size)) + } else { + mb.literal_context_map = mb.literal_context_map[:mb.literal_context_map_size] + } for i = 0; i < mb.literal_split.num_types; i++ { var offset uint32 = uint32(i * num_contexts) @@ -466,7 +487,7 @@ func mapStaticContexts(num_contexts uint, static_context_map []uint32, mb *metaB } } -func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) { +func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) { var lit_blocks struct { plain blockSplitterLiteral ctx contextBlockSplitter @@ -474,8 +495,7 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b var cmd_blocks blockSplitterCommand var dist_blocks blockSplitterDistance var num_literals uint = 0 - var i uint - for i = 0; i < n_commands; i++ { + for i := range commands { num_literals += uint(commands[i].insert_len_) } @@ -485,11 +505,10 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b initContextBlockSplitter(&lit_blocks.ctx, 256, num_contexts, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size) } - initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, n_commands, &mb.command_split, &mb.command_histograms, &mb.command_histograms_size) - initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, n_commands, &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size) + initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, uint(len(commands)), &mb.command_split, &mb.command_histograms, &mb.command_histograms_size) + initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, uint(len(commands)), &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size) - for i = 0; i < n_commands; i++ { - var cmd command = commands[i] + for _, cmd := range commands { var j uint blockSplitterAddSymbolCommand(&cmd_blocks, uint(cmd.cmd_prefix_)) for j = uint(cmd.insert_len_); j != 0; j-- { @@ -530,11 +549,11 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b } } -func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) { +func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) { if num_contexts == 1 { - buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, n_commands, mb) + buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, mb) } else { - buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, n_commands, mb) + buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, mb) } } diff --git a/vendor/github.com/andybalholm/brotli/metablock_command.go b/vendor/github.com/andybalholm/brotli/metablock_command.go index d47541c5..14c7b771 100644 --- a/vendor/github.com/andybalholm/brotli/metablock_command.go +++ b/vendor/github.com/andybalholm/brotli/metablock_command.go @@ -43,9 +43,12 @@ func initBlockSplitterCommand(self *blockSplitterCommand, alphabet_size uint, mi brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks) brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks) self.split_.num_blocks = max_num_blocks - assert(*histograms == nil) *histograms_size = max_num_types - *histograms = make([]histogramCommand, (*histograms_size)) + if histograms == nil || cap(*histograms) < int(*histograms_size) { + *histograms = make([]histogramCommand, (*histograms_size)) + } else { + *histograms = (*histograms)[:*histograms_size] + } self.histograms_ = *histograms /* Clear only current histogram. */ diff --git a/vendor/github.com/andybalholm/brotli/metablock_distance.go b/vendor/github.com/andybalholm/brotli/metablock_distance.go index 95923127..5110a810 100644 --- a/vendor/github.com/andybalholm/brotli/metablock_distance.go +++ b/vendor/github.com/andybalholm/brotli/metablock_distance.go @@ -43,9 +43,12 @@ func initBlockSplitterDistance(self *blockSplitterDistance, alphabet_size uint, brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks) brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks) self.split_.num_blocks = max_num_blocks - assert(*histograms == nil) *histograms_size = max_num_types - *histograms = make([]histogramDistance, (*histograms_size)) + if histograms == nil || cap(*histograms) < int(*histograms_size) { + *histograms = make([]histogramDistance, *histograms_size) + } else { + *histograms = (*histograms)[:*histograms_size] + } self.histograms_ = *histograms /* Clear only current histogram. */ diff --git a/vendor/github.com/andybalholm/brotli/metablock_literal.go b/vendor/github.com/andybalholm/brotli/metablock_literal.go index d7e8a7c9..307f8da8 100644 --- a/vendor/github.com/andybalholm/brotli/metablock_literal.go +++ b/vendor/github.com/andybalholm/brotli/metablock_literal.go @@ -43,9 +43,12 @@ func initBlockSplitterLiteral(self *blockSplitterLiteral, alphabet_size uint, mi brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks) brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks) self.split_.num_blocks = max_num_blocks - assert(*histograms == nil) *histograms_size = max_num_types - *histograms = make([]histogramLiteral, (*histograms_size)) + if histograms == nil || cap(*histograms) < int(*histograms_size) { + *histograms = make([]histogramLiteral, *histograms_size) + } else { + *histograms = (*histograms)[:*histograms_size] + } self.histograms_ = *histograms /* Clear only current histogram. */ diff --git a/vendor/github.com/andybalholm/brotli/reader.go b/vendor/github.com/andybalholm/brotli/reader.go index 5c795e6e..9419c79c 100644 --- a/vendor/github.com/andybalholm/brotli/reader.go +++ b/vendor/github.com/andybalholm/brotli/reader.go @@ -27,13 +27,21 @@ func NewReader(src io.Reader) *Reader { } // Reset discards the Reader's state and makes it equivalent to the result of -// its original state from NewReader, but writing to src instead. +// its original state from NewReader, but reading from src instead. // This permits reusing a Reader rather than allocating a new one. // Error is always nil func (r *Reader) Reset(src io.Reader) error { + if r.error_code < 0 { + // There was an unrecoverable error, leaving the Reader's state + // undefined. Clear out everything but the buffer. + *r = Reader{buf: r.buf} + } + decoderStateInit(r) r.src = src - r.buf = make([]byte, readBufSize) + if r.buf == nil { + r.buf = make([]byte, readBufSize) + } return nil } diff --git a/vendor/github.com/andybalholm/brotli/ringbuffer.go b/vendor/github.com/andybalholm/brotli/ringbuffer.go index 693a3f65..1c8f86fe 100644 --- a/vendor/github.com/andybalholm/brotli/ringbuffer.go +++ b/vendor/github.com/andybalholm/brotli/ringbuffer.go @@ -27,10 +27,7 @@ type ringBuffer struct { } func ringBufferInit(rb *ringBuffer) { - rb.cur_size_ = 0 rb.pos_ = 0 - rb.data_ = nil - rb.buffer_ = nil } func ringBufferSetup(params *encoderParams, rb *ringBuffer) { @@ -47,11 +44,16 @@ const kSlackForEightByteHashingEverywhere uint = 7 /* Allocates or re-allocates data_ to the given length + plus some slack region before and after. Fills the slack regions with zeros. */ func ringBufferInitBuffer(buflen uint32, rb *ringBuffer) { - var new_data []byte = make([]byte, (2 + uint(buflen) + kSlackForEightByteHashingEverywhere)) + var new_data []byte var i uint + size := 2 + int(buflen) + int(kSlackForEightByteHashingEverywhere) + if cap(rb.data_) < size { + new_data = make([]byte, size) + } else { + new_data = rb.data_[:size] + } if rb.data_ != nil { copy(new_data, rb.data_[:2+rb.cur_size_+uint32(kSlackForEightByteHashingEverywhere)]) - rb.data_ = nil } rb.data_ = new_data diff --git a/vendor/github.com/andybalholm/brotli/state.go b/vendor/github.com/andybalholm/brotli/state.go index d03348fe..38d753eb 100644 --- a/vendor/github.com/andybalholm/brotli/state.go +++ b/vendor/github.com/andybalholm/brotli/state.go @@ -200,7 +200,6 @@ func decoderStateInit(s *Reader) bool { s.block_type_trees = nil s.block_len_trees = nil - s.ringbuffer = nil s.ringbuffer_size = 0 s.new_ringbuffer_size = 0 s.ringbuffer_mask = 0 diff --git a/vendor/github.com/andybalholm/brotli/static_dict.go b/vendor/github.com/andybalholm/brotli/static_dict.go index 8e7492d7..bc05566d 100644 --- a/vendor/github.com/andybalholm/brotli/static_dict.go +++ b/vendor/github.com/andybalholm/brotli/static_dict.go @@ -77,8 +77,7 @@ func findAllStaticDictionaryMatches(dict *encoderDictionary, data []byte, min_le var offset uint = uint(dict.buckets[hash(data)]) var end bool = offset == 0 for !end { - var w dictWord - w = dict.dict_words[offset] + w := dict.dict_words[offset] offset++ var l uint = uint(w.len) & 0x1F var n uint = uint(1) << dict.words.size_bits_by_length[l] @@ -431,8 +430,7 @@ func findAllStaticDictionaryMatches(dict *encoderDictionary, data []byte, min_le var offset uint = uint(dict.buckets[hash(data[1:])]) var end bool = offset == 0 for !end { - var w dictWord - w = dict.dict_words[offset] + w := dict.dict_words[offset] offset++ var l uint = uint(w.len) & 0x1F var n uint = uint(1) << dict.words.size_bits_by_length[l] @@ -596,8 +594,7 @@ func findAllStaticDictionaryMatches(dict *encoderDictionary, data []byte, min_le var offset uint = uint(dict.buckets[hash(data[2:])]) var end bool = offset == 0 for !end { - var w dictWord - w = dict.dict_words[offset] + w := dict.dict_words[offset] offset++ var l uint = uint(w.len) & 0x1F var n uint = uint(1) << dict.words.size_bits_by_length[l] @@ -629,8 +626,7 @@ func findAllStaticDictionaryMatches(dict *encoderDictionary, data []byte, min_le var offset uint = uint(dict.buckets[hash(data[5:])]) var end bool = offset == 0 for !end { - var w dictWord - w = dict.dict_words[offset] + w := dict.dict_words[offset] offset++ var l uint = uint(w.len) & 0x1F var n uint = uint(1) << dict.words.size_bits_by_length[l] diff --git a/vendor/github.com/andybalholm/brotli/utf8_util.go b/vendor/github.com/andybalholm/brotli/utf8_util.go index f86de3d2..3244247e 100644 --- a/vendor/github.com/andybalholm/brotli/utf8_util.go +++ b/vendor/github.com/andybalholm/brotli/utf8_util.go @@ -58,8 +58,7 @@ func isMostlyUTF8(data []byte, pos uint, mask uint, length uint, min_fraction fl var i uint = 0 for i < length { var symbol int - var current_data []byte - current_data = data[(pos+i)&mask:] + current_data := data[(pos+i)&mask:] var bytes_read uint = parseAsUTF8(&symbol, current_data, length-i) i += bytes_read if symbol < 0x110000 { diff --git a/vendor/github.com/andybalholm/brotli/write_bits.go b/vendor/github.com/andybalholm/brotli/write_bits.go index 8f15c202..87299011 100644 --- a/vendor/github.com/andybalholm/brotli/write_bits.go +++ b/vendor/github.com/andybalholm/brotli/write_bits.go @@ -1,5 +1,7 @@ package brotli +import "encoding/binary" + /* Copyright 2010 Google Inc. All Rights Reserved. Distributed under MIT license. @@ -24,21 +26,15 @@ package brotli For n bits, we take the last 5 bits, OR that with high bits in BYTE-0, and locate the rest in BYTE+1, BYTE+2, etc. */ func writeBits(n_bits uint, bits uint64, pos *uint, array []byte) { - var array_pos []byte = array[*pos>>3:] - var bits_reserved_in_first_byte uint = (*pos & 7) - /* implicit & 0xFF is assumed for uint8_t arithmetics */ - - var bits_left_to_write uint - bits <<= bits_reserved_in_first_byte - array_pos[0] |= byte(bits) - array_pos = array_pos[1:] - for bits_left_to_write = n_bits + bits_reserved_in_first_byte; bits_left_to_write >= 9; bits_left_to_write -= 8 { - bits >>= 8 - array_pos[0] = byte(bits) - array_pos = array_pos[1:] - } - - array_pos[0] = 0 + /* This branch of the code can write up to 56 bits at a time, + 7 bits are lost by being perhaps already in *p and at least + 1 bit is needed to initialize the bit-stream ahead (i.e. if 7 + bits are in *p and we write 57 bits, then the next write will + access a byte that was never initialized). */ + p := array[*pos>>3:] + v := uint64(p[0]) + v |= bits << (*pos & 7) + binary.LittleEndian.PutUint64(p, v) *pos += n_bits } diff --git a/vendor/github.com/andybalholm/brotli/writer.go b/vendor/github.com/andybalholm/brotli/writer.go index 92c128c4..8a688117 100644 --- a/vendor/github.com/andybalholm/brotli/writer.go +++ b/vendor/github.com/andybalholm/brotli/writer.go @@ -1,12 +1,10 @@ package brotli import ( - "compress/gzip" "errors" "io" - "net/http" - "github.com/golang/gddo/httputil" + "github.com/andybalholm/brotli/matchfinder" ) const ( @@ -50,11 +48,8 @@ func NewWriterLevel(dst io.Writer, level int) *Writer { // NewWriterOptions is like NewWriter but specifies WriterOptions func NewWriterOptions(dst io.Writer, options WriterOptions) *Writer { w := new(Writer) + w.options = options w.Reset(dst) - w.params.quality = options.Quality - if options.LGWin > 0 { - w.params.lgwin = uint(options.LGWin) - } return w } @@ -63,13 +58,21 @@ func NewWriterOptions(dst io.Writer, options WriterOptions) *Writer { // instead. This permits reusing a Writer rather than allocating a new one. func (w *Writer) Reset(dst io.Writer) { encoderInitState(w) + w.params.quality = w.options.Quality + if w.options.LGWin > 0 { + w.params.lgwin = uint(w.options.LGWin) + } w.dst = dst + w.err = nil } func (w *Writer) writeChunk(p []byte, op int) (n int, err error) { if w.dst == nil { return 0, errWriterClosed } + if w.err != nil { + return 0, w.err + } for { availableIn := uint(len(p)) @@ -82,16 +85,8 @@ func (w *Writer) writeChunk(p []byte, op int) (n int, err error) { return n, errEncode } - outputData := encoderTakeOutput(w) - - if len(outputData) > 0 { - _, err = w.dst.Write(outputData) - if err != nil { - return n, err - } - } - if len(p) == 0 { - return n, nil + if len(p) == 0 || w.err != nil { + return n, w.err } } } @@ -125,31 +120,43 @@ type nopCloser struct { func (nopCloser) Close() error { return nil } -// HTTPCompressor chooses a compression method (brotli, gzip, or none) based on -// the Accept-Encoding header, sets the Content-Encoding header, and returns a -// WriteCloser that implements that compression. The Close method must be called -// before the current HTTP handler returns. -// -// Due to https://github.com/golang/go/issues/31753, the response will not be -// compressed unless you set a Content-Type header before you call -// HTTPCompressor. -func HTTPCompressor(w http.ResponseWriter, r *http.Request) io.WriteCloser { - if w.Header().Get("Content-Type") == "" { - return nopCloser{w} - } - - if w.Header().Get("Vary") == "" { - w.Header().Set("Vary", "Accept-Encoding") +// NewWriterV2 is like NewWriterLevel, but it uses the new implementation +// based on the matchfinder package. It currently supports up to level 7; +// if a higher level is specified, level 7 will be used. +func NewWriterV2(dst io.Writer, level int) *matchfinder.Writer { + var mf matchfinder.MatchFinder + if level < 2 { + mf = matchfinder.M0{Lazy: level == 1} + } else { + hashLen := 6 + if level >= 6 { + hashLen = 5 + } + chainLen := 64 + switch level { + case 2: + chainLen = 0 + case 3: + chainLen = 1 + case 4: + chainLen = 2 + case 5: + chainLen = 4 + case 6: + chainLen = 8 + } + mf = &matchfinder.M4{ + MaxDistance: 1 << 20, + ChainLength: chainLen, + HashLen: hashLen, + DistanceBitCost: 57, + } } - encoding := httputil.NegotiateContentEncoding(r, []string{"br", "gzip"}) - switch encoding { - case "br": - w.Header().Set("Content-Encoding", "br") - return NewWriter(w) - case "gzip": - w.Header().Set("Content-Encoding", "gzip") - return gzip.NewWriter(w) + return &matchfinder.Writer{ + Dest: dst, + MatchFinder: mf, + Encoder: &Encoder{}, + BlockSize: 1 << 16, } - return nopCloser{w} } diff --git a/vendor/github.com/golang/gddo/LICENSE b/vendor/github.com/golang/gddo/LICENSE deleted file mode 100644 index 65d761bc..00000000 --- a/vendor/github.com/golang/gddo/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2013 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/golang/gddo/httputil/buster.go b/vendor/github.com/golang/gddo/httputil/buster.go deleted file mode 100644 index beab151a..00000000 --- a/vendor/github.com/golang/gddo/httputil/buster.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd. - -package httputil - -import ( - "io" - "io/ioutil" - "net/http" - "net/url" - "strings" - "sync" -) - -type busterWriter struct { - headerMap http.Header - status int - io.Writer -} - -func (bw *busterWriter) Header() http.Header { - return bw.headerMap -} - -func (bw *busterWriter) WriteHeader(status int) { - bw.status = status -} - -// CacheBusters maintains a cache of cache busting tokens for static resources served by Handler. -type CacheBusters struct { - Handler http.Handler - - mu sync.Mutex - tokens map[string]string -} - -func sanitizeTokenRune(r rune) rune { - if r <= ' ' || r >= 127 { - return -1 - } - // Convert percent encoding reserved characters to '-'. - if strings.ContainsRune("!#$&'()*+,/:;=?@[]", r) { - return '-' - } - return r -} - -// Get returns the cache busting token for path. If the token is not already -// cached, Get issues a HEAD request on handler and uses the response ETag and -// Last-Modified headers to compute a token. -func (cb *CacheBusters) Get(path string) string { - cb.mu.Lock() - if cb.tokens == nil { - cb.tokens = make(map[string]string) - } - token, ok := cb.tokens[path] - cb.mu.Unlock() - if ok { - return token - } - - w := busterWriter{ - Writer: ioutil.Discard, - headerMap: make(http.Header), - } - r := &http.Request{URL: &url.URL{Path: path}, Method: "HEAD"} - cb.Handler.ServeHTTP(&w, r) - - if w.status == 200 { - token = w.headerMap.Get("Etag") - if token == "" { - token = w.headerMap.Get("Last-Modified") - } - token = strings.Trim(token, `" `) - token = strings.Map(sanitizeTokenRune, token) - } - - cb.mu.Lock() - cb.tokens[path] = token - cb.mu.Unlock() - - return token -} - -// AppendQueryParam appends the token as a query parameter to path. -func (cb *CacheBusters) AppendQueryParam(path string, name string) string { - token := cb.Get(path) - if token == "" { - return path - } - return path + "?" + name + "=" + token -} diff --git a/vendor/github.com/golang/gddo/httputil/header/header.go b/vendor/github.com/golang/gddo/httputil/header/header.go deleted file mode 100644 index 0f1572e3..00000000 --- a/vendor/github.com/golang/gddo/httputil/header/header.go +++ /dev/null @@ -1,298 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd. - -// Package header provides functions for parsing HTTP headers. -package header - -import ( - "net/http" - "strings" - "time" -) - -// Octet types from RFC 2616. -var octetTypes [256]octetType - -type octetType byte - -const ( - isToken octetType = 1 << iota - isSpace -) - -func init() { - // OCTET = - // CHAR = - // CTL = - // CR = - // LF = - // SP = - // HT = - // <"> = - // CRLF = CR LF - // LWS = [CRLF] 1*( SP | HT ) - // TEXT = - // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> - // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT - // token = 1* - // qdtext = > - - for c := 0; c < 256; c++ { - var t octetType - isCtl := c <= 31 || c == 127 - isChar := 0 <= c && c <= 127 - isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0 - if strings.IndexRune(" \t\r\n", rune(c)) >= 0 { - t |= isSpace - } - if isChar && !isCtl && !isSeparator { - t |= isToken - } - octetTypes[c] = t - } -} - -// Copy returns a shallow copy of the header. -func Copy(header http.Header) http.Header { - h := make(http.Header) - for k, vs := range header { - h[k] = vs - } - return h -} - -var timeLayouts = []string{"Mon, 02 Jan 2006 15:04:05 GMT", time.RFC850, time.ANSIC} - -// ParseTime parses the header as time. The zero value is returned if the -// header is not present or there is an error parsing the -// header. -func ParseTime(header http.Header, key string) time.Time { - if s := header.Get(key); s != "" { - for _, layout := range timeLayouts { - if t, err := time.Parse(layout, s); err == nil { - return t.UTC() - } - } - } - return time.Time{} -} - -// ParseList parses a comma separated list of values. Commas are ignored in -// quoted strings. Quoted values are not unescaped or unquoted. Whitespace is -// trimmed. -func ParseList(header http.Header, key string) []string { - var result []string - for _, s := range header[http.CanonicalHeaderKey(key)] { - begin := 0 - end := 0 - escape := false - quote := false - for i := 0; i < len(s); i++ { - b := s[i] - switch { - case escape: - escape = false - end = i + 1 - case quote: - switch b { - case '\\': - escape = true - case '"': - quote = false - } - end = i + 1 - case b == '"': - quote = true - end = i + 1 - case octetTypes[b]&isSpace != 0: - if begin == end { - begin = i + 1 - end = begin - } - case b == ',': - if begin < end { - result = append(result, s[begin:end]) - } - begin = i + 1 - end = begin - default: - end = i + 1 - } - } - if begin < end { - result = append(result, s[begin:end]) - } - } - return result -} - -// ParseValueAndParams parses a comma separated list of values with optional -// semicolon separated name-value pairs. Content-Type and Content-Disposition -// headers are in this format. -func ParseValueAndParams(header http.Header, key string) (value string, params map[string]string) { - params = make(map[string]string) - s := header.Get(key) - value, s = expectTokenSlash(s) - if value == "" { - return - } - value = strings.ToLower(value) - s = skipSpace(s) - for strings.HasPrefix(s, ";") { - var pkey string - pkey, s = expectToken(skipSpace(s[1:])) - if pkey == "" { - return - } - if !strings.HasPrefix(s, "=") { - return - } - var pvalue string - pvalue, s = expectTokenOrQuoted(s[1:]) - if pvalue == "" { - return - } - pkey = strings.ToLower(pkey) - params[pkey] = pvalue - s = skipSpace(s) - } - return -} - -// AcceptSpec describes an Accept* header. -type AcceptSpec struct { - Value string - Q float64 -} - -// ParseAccept parses Accept* headers. -func ParseAccept(header http.Header, key string) (specs []AcceptSpec) { -loop: - for _, s := range header[key] { - for { - var spec AcceptSpec - spec.Value, s = expectTokenSlash(s) - if spec.Value == "" { - continue loop - } - spec.Q = 1.0 - s = skipSpace(s) - if strings.HasPrefix(s, ";") { - s = skipSpace(s[1:]) - if !strings.HasPrefix(s, "q=") { - continue loop - } - spec.Q, s = expectQuality(s[2:]) - if spec.Q < 0.0 { - continue loop - } - } - specs = append(specs, spec) - s = skipSpace(s) - if !strings.HasPrefix(s, ",") { - continue loop - } - s = skipSpace(s[1:]) - } - } - return -} - -func skipSpace(s string) (rest string) { - i := 0 - for ; i < len(s); i++ { - if octetTypes[s[i]]&isSpace == 0 { - break - } - } - return s[i:] -} - -func expectToken(s string) (token, rest string) { - i := 0 - for ; i < len(s); i++ { - if octetTypes[s[i]]&isToken == 0 { - break - } - } - return s[:i], s[i:] -} - -func expectTokenSlash(s string) (token, rest string) { - i := 0 - for ; i < len(s); i++ { - b := s[i] - if (octetTypes[b]&isToken == 0) && b != '/' { - break - } - } - return s[:i], s[i:] -} - -func expectQuality(s string) (q float64, rest string) { - switch { - case len(s) == 0: - return -1, "" - case s[0] == '0': - q = 0 - case s[0] == '1': - q = 1 - default: - return -1, "" - } - s = s[1:] - if !strings.HasPrefix(s, ".") { - return q, s - } - s = s[1:] - i := 0 - n := 0 - d := 1 - for ; i < len(s); i++ { - b := s[i] - if b < '0' || b > '9' { - break - } - n = n*10 + int(b) - '0' - d *= 10 - } - return q + float64(n)/float64(d), s[i:] -} - -func expectTokenOrQuoted(s string) (value string, rest string) { - if !strings.HasPrefix(s, "\"") { - return expectToken(s) - } - s = s[1:] - for i := 0; i < len(s); i++ { - switch s[i] { - case '"': - return s[:i], s[i+1:] - case '\\': - p := make([]byte, len(s)-1) - j := copy(p, s[:i]) - escape := true - for i = i + 1; i < len(s); i++ { - b := s[i] - switch { - case escape: - escape = false - p[j] = b - j++ - case b == '\\': - escape = true - case b == '"': - return string(p[:j]), s[i+1:] - default: - p[j] = b - j++ - } - } - return "", "" - } - } - return "", "" -} diff --git a/vendor/github.com/golang/gddo/httputil/httputil.go b/vendor/github.com/golang/gddo/httputil/httputil.go deleted file mode 100644 index a03717c0..00000000 --- a/vendor/github.com/golang/gddo/httputil/httputil.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd. - -// Package httputil is a toolkit for the Go net/http package. -package httputil - -import ( - "net" - "net/http" -) - -// StripPort removes the port specification from an address. -func StripPort(s string) string { - if h, _, err := net.SplitHostPort(s); err == nil { - s = h - } - return s -} - -// Error defines a type for a function that accepts a ResponseWriter for -// a Request with the HTTP status code and error. -type Error func(w http.ResponseWriter, r *http.Request, status int, err error) diff --git a/vendor/github.com/golang/gddo/httputil/negotiate.go b/vendor/github.com/golang/gddo/httputil/negotiate.go deleted file mode 100644 index 6af3e4ca..00000000 --- a/vendor/github.com/golang/gddo/httputil/negotiate.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd. - -package httputil - -import ( - "github.com/golang/gddo/httputil/header" - "net/http" - "strings" -) - -// NegotiateContentEncoding returns the best offered content encoding for the -// request's Accept-Encoding header. If two offers match with equal weight and -// then the offer earlier in the list is preferred. If no offers are -// acceptable, then "" is returned. -func NegotiateContentEncoding(r *http.Request, offers []string) string { - bestOffer := "identity" - bestQ := -1.0 - specs := header.ParseAccept(r.Header, "Accept-Encoding") - for _, offer := range offers { - for _, spec := range specs { - if spec.Q > bestQ && - (spec.Value == "*" || spec.Value == offer) { - bestQ = spec.Q - bestOffer = offer - } - } - } - if bestQ == 0 { - bestOffer = "" - } - return bestOffer -} - -// NegotiateContentType returns the best offered content type for the request's -// Accept header. If two offers match with equal weight, then the more specific -// offer is preferred. For example, text/* trumps */*. If two offers match -// with equal weight and specificity, then the offer earlier in the list is -// preferred. If no offers match, then defaultOffer is returned. -func NegotiateContentType(r *http.Request, offers []string, defaultOffer string) string { - bestOffer := defaultOffer - bestQ := -1.0 - bestWild := 3 - specs := header.ParseAccept(r.Header, "Accept") - for _, offer := range offers { - for _, spec := range specs { - switch { - case spec.Q == 0.0: - // ignore - case spec.Q < bestQ: - // better match found - case spec.Value == "*/*": - if spec.Q > bestQ || bestWild > 2 { - bestQ = spec.Q - bestWild = 2 - bestOffer = offer - } - case strings.HasSuffix(spec.Value, "/*"): - if strings.HasPrefix(offer, spec.Value[:len(spec.Value)-1]) && - (spec.Q > bestQ || bestWild > 1) { - bestQ = spec.Q - bestWild = 1 - bestOffer = offer - } - default: - if spec.Value == offer && - (spec.Q > bestQ || bestWild > 0) { - bestQ = spec.Q - bestWild = 0 - bestOffer = offer - } - } - } - } - return bestOffer -} diff --git a/vendor/github.com/golang/gddo/httputil/respbuf.go b/vendor/github.com/golang/gddo/httputil/respbuf.go deleted file mode 100644 index 051af211..00000000 --- a/vendor/github.com/golang/gddo/httputil/respbuf.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd. - -package httputil - -import ( - "bytes" - "net/http" - "strconv" -) - -// ResponseBuffer is the current response being composed by its owner. -// It implements http.ResponseWriter and io.WriterTo. -type ResponseBuffer struct { - buf bytes.Buffer - status int - header http.Header -} - -// Write implements the http.ResponseWriter interface. -func (rb *ResponseBuffer) Write(p []byte) (int, error) { - return rb.buf.Write(p) -} - -// WriteHeader implements the http.ResponseWriter interface. -func (rb *ResponseBuffer) WriteHeader(status int) { - rb.status = status -} - -// Header implements the http.ResponseWriter interface. -func (rb *ResponseBuffer) Header() http.Header { - if rb.header == nil { - rb.header = make(http.Header) - } - return rb.header -} - -// WriteTo implements the io.WriterTo interface. -func (rb *ResponseBuffer) WriteTo(w http.ResponseWriter) error { - for k, v := range rb.header { - w.Header()[k] = v - } - if rb.buf.Len() > 0 { - w.Header().Set("Content-Length", strconv.Itoa(rb.buf.Len())) - } - if rb.status != 0 { - w.WriteHeader(rb.status) - } - if rb.buf.Len() > 0 { - if _, err := w.Write(rb.buf.Bytes()); err != nil { - return err - } - } - return nil -} diff --git a/vendor/github.com/golang/gddo/httputil/static.go b/vendor/github.com/golang/gddo/httputil/static.go deleted file mode 100644 index 6610dde0..00000000 --- a/vendor/github.com/golang/gddo/httputil/static.go +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd. - -package httputil - -import ( - "bytes" - "crypto/sha1" - "errors" - "fmt" - "github.com/golang/gddo/httputil/header" - "io" - "io/ioutil" - "mime" - "net/http" - "os" - "path" - "path/filepath" - "strconv" - "strings" - "sync" - "time" -) - -// StaticServer serves static files. -type StaticServer struct { - // Dir specifies the location of the directory containing the files to serve. - Dir string - - // MaxAge specifies the maximum age for the cache control and expiration - // headers. - MaxAge time.Duration - - // Error specifies the function used to generate error responses. If Error - // is nil, then http.Error is used to generate error responses. - Error Error - - // MIMETypes is a map from file extensions to MIME types. - MIMETypes map[string]string - - mu sync.Mutex - etags map[string]string -} - -func (ss *StaticServer) resolve(fname string) string { - if path.IsAbs(fname) { - panic("Absolute path not allowed when creating a StaticServer handler") - } - dir := ss.Dir - if dir == "" { - dir = "." - } - fname = filepath.FromSlash(fname) - return filepath.Join(dir, fname) -} - -func (ss *StaticServer) mimeType(fname string) string { - ext := path.Ext(fname) - var mimeType string - if ss.MIMETypes != nil { - mimeType = ss.MIMETypes[ext] - } - if mimeType == "" { - mimeType = mime.TypeByExtension(ext) - } - if mimeType == "" { - mimeType = "application/octet-stream" - } - return mimeType -} - -func (ss *StaticServer) openFile(fname string) (io.ReadCloser, int64, string, error) { - f, err := os.Open(fname) - if err != nil { - return nil, 0, "", err - } - fi, err := f.Stat() - if err != nil { - f.Close() - return nil, 0, "", err - } - const modeType = os.ModeDir | os.ModeSymlink | os.ModeNamedPipe | os.ModeSocket | os.ModeDevice - if fi.Mode()&modeType != 0 { - f.Close() - return nil, 0, "", errors.New("not a regular file") - } - return f, fi.Size(), ss.mimeType(fname), nil -} - -// FileHandler returns a handler that serves a single file. The file is -// specified by a slash separated path relative to the static server's Dir -// field. -func (ss *StaticServer) FileHandler(fileName string) http.Handler { - id := fileName - fileName = ss.resolve(fileName) - return &staticHandler{ - ss: ss, - id: func(_ string) string { return id }, - open: func(_ string) (io.ReadCloser, int64, string, error) { return ss.openFile(fileName) }, - } -} - -// DirectoryHandler returns a handler that serves files from a directory tree. -// The directory is specified by a slash separated path relative to the static -// server's Dir field. -func (ss *StaticServer) DirectoryHandler(prefix, dirName string) http.Handler { - if !strings.HasSuffix(prefix, "/") { - prefix += "/" - } - idBase := dirName - dirName = ss.resolve(dirName) - return &staticHandler{ - ss: ss, - id: func(p string) string { - if !strings.HasPrefix(p, prefix) { - return "." - } - return path.Join(idBase, p[len(prefix):]) - }, - open: func(p string) (io.ReadCloser, int64, string, error) { - if !strings.HasPrefix(p, prefix) { - return nil, 0, "", errors.New("request url does not match directory prefix") - } - p = p[len(prefix):] - return ss.openFile(filepath.Join(dirName, filepath.FromSlash(p))) - }, - } -} - -// FilesHandler returns a handler that serves the concatentation of the -// specified files. The files are specified by slash separated paths relative -// to the static server's Dir field. -func (ss *StaticServer) FilesHandler(fileNames ...string) http.Handler { - - // todo: cache concatenated files on disk and serve from there. - - mimeType := ss.mimeType(fileNames[0]) - var buf []byte - var openErr error - - for _, fileName := range fileNames { - p, err := ioutil.ReadFile(ss.resolve(fileName)) - if err != nil { - openErr = err - buf = nil - break - } - buf = append(buf, p...) - } - - id := strings.Join(fileNames, " ") - - return &staticHandler{ - ss: ss, - id: func(_ string) string { return id }, - open: func(p string) (io.ReadCloser, int64, string, error) { - return ioutil.NopCloser(bytes.NewReader(buf)), int64(len(buf)), mimeType, openErr - }, - } -} - -type staticHandler struct { - id func(fname string) string - open func(p string) (io.ReadCloser, int64, string, error) - ss *StaticServer -} - -func (h *staticHandler) error(w http.ResponseWriter, r *http.Request, status int, err error) { - http.Error(w, http.StatusText(status), status) -} - -func (h *staticHandler) etag(p string) (string, error) { - id := h.id(p) - - h.ss.mu.Lock() - if h.ss.etags == nil { - h.ss.etags = make(map[string]string) - } - etag := h.ss.etags[id] - h.ss.mu.Unlock() - - if etag != "" { - return etag, nil - } - - // todo: if a concurrent goroutine is calculating the hash, then wait for - // it instead of computing it again here. - - rc, _, _, err := h.open(p) - if err != nil { - return "", err - } - - defer rc.Close() - - w := sha1.New() - _, err = io.Copy(w, rc) - if err != nil { - return "", err - } - - etag = fmt.Sprintf(`"%x"`, w.Sum(nil)) - - h.ss.mu.Lock() - h.ss.etags[id] = etag - h.ss.mu.Unlock() - - return etag, nil -} - -func (h *staticHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - p := path.Clean(r.URL.Path) - if p != r.URL.Path { - http.Redirect(w, r, p, 301) - return - } - - etag, err := h.etag(p) - if err != nil { - h.error(w, r, http.StatusNotFound, err) - return - } - - maxAge := h.ss.MaxAge - if maxAge == 0 { - maxAge = 24 * time.Hour - } - if r.FormValue("v") != "" { - maxAge = 365 * 24 * time.Hour - } - - cacheControl := fmt.Sprintf("public, max-age=%d", maxAge/time.Second) - - for _, e := range header.ParseList(r.Header, "If-None-Match") { - if e == etag { - w.Header().Set("Cache-Control", cacheControl) - w.Header().Set("Etag", etag) - w.WriteHeader(http.StatusNotModified) - return - } - } - - rc, cl, ct, err := h.open(p) - if err != nil { - h.error(w, r, http.StatusNotFound, err) - return - } - defer rc.Close() - - w.Header().Set("Cache-Control", cacheControl) - w.Header().Set("Etag", etag) - if ct != "" { - w.Header().Set("Content-Type", ct) - } - if cl != 0 { - w.Header().Set("Content-Length", strconv.FormatInt(cl, 10)) - } - w.WriteHeader(http.StatusOK) - if r.Method != "HEAD" { - io.Copy(w, rc) - } -} diff --git a/vendor/github.com/golang/gddo/httputil/transport.go b/vendor/github.com/golang/gddo/httputil/transport.go deleted file mode 100644 index fdad3b47..00000000 --- a/vendor/github.com/golang/gddo/httputil/transport.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd. - -// This file implements a http.RoundTripper that authenticates -// requests issued against api.github.com endpoint. - -package httputil - -import ( - "net/http" - "net/url" -) - -// AuthTransport is an implementation of http.RoundTripper that authenticates -// with the GitHub API. -// -// When both a token and client credentials are set, the latter is preferred. -type AuthTransport struct { - UserAgent string - GithubToken string - GithubClientID string - GithubClientSecret string - Base http.RoundTripper -} - -// RoundTrip implements the http.RoundTripper interface. -func (t *AuthTransport) RoundTrip(req *http.Request) (*http.Response, error) { - var reqCopy *http.Request - if t.UserAgent != "" { - reqCopy = copyRequest(req) - reqCopy.Header.Set("User-Agent", t.UserAgent) - } - if req.URL.Host == "api.github.com" && req.URL.Scheme == "https" { - switch { - case t.GithubClientID != "" && t.GithubClientSecret != "": - if reqCopy == nil { - reqCopy = copyRequest(req) - } - if reqCopy.URL.RawQuery == "" { - reqCopy.URL.RawQuery = "client_id=" + t.GithubClientID + "&client_secret=" + t.GithubClientSecret - } else { - reqCopy.URL.RawQuery += "&client_id=" + t.GithubClientID + "&client_secret=" + t.GithubClientSecret - } - case t.GithubToken != "": - if reqCopy == nil { - reqCopy = copyRequest(req) - } - reqCopy.Header.Set("Authorization", "token "+t.GithubToken) - } - } - if reqCopy != nil { - return t.base().RoundTrip(reqCopy) - } - return t.base().RoundTrip(req) -} - -// CancelRequest cancels an in-flight request by closing its connection. -func (t *AuthTransport) CancelRequest(req *http.Request) { - type canceler interface { - CancelRequest(req *http.Request) - } - if cr, ok := t.base().(canceler); ok { - cr.CancelRequest(req) - } -} - -func (t *AuthTransport) base() http.RoundTripper { - if t.Base != nil { - return t.Base - } - return http.DefaultTransport -} - -func copyRequest(req *http.Request) *http.Request { - req2 := new(http.Request) - *req2 = *req - req2.URL = new(url.URL) - *req2.URL = *req.URL - req2.Header = make(http.Header, len(req.Header)) - for k, s := range req.Header { - req2.Header[k] = append([]string(nil), s...) - } - return req2 -} diff --git a/vendor/github.com/klauspost/compress/gzip/gunzip.go b/vendor/github.com/klauspost/compress/gzip/gunzip.go new file mode 100644 index 00000000..6d630c39 --- /dev/null +++ b/vendor/github.com/klauspost/compress/gzip/gunzip.go @@ -0,0 +1,374 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gzip implements reading and writing of gzip format compressed files, +// as specified in RFC 1952. +package gzip + +import ( + "bufio" + "compress/gzip" + "encoding/binary" + "hash/crc32" + "io" + "time" + + "github.com/klauspost/compress/flate" +) + +const ( + gzipID1 = 0x1f + gzipID2 = 0x8b + gzipDeflate = 8 + flagText = 1 << 0 + flagHdrCrc = 1 << 1 + flagExtra = 1 << 2 + flagName = 1 << 3 + flagComment = 1 << 4 +) + +var ( + // ErrChecksum is returned when reading GZIP data that has an invalid checksum. + ErrChecksum = gzip.ErrChecksum + // ErrHeader is returned when reading GZIP data that has an invalid header. + ErrHeader = gzip.ErrHeader +) + +var le = binary.LittleEndian + +// noEOF converts io.EOF to io.ErrUnexpectedEOF. +func noEOF(err error) error { + if err == io.EOF { + return io.ErrUnexpectedEOF + } + return err +} + +// The gzip file stores a header giving metadata about the compressed file. +// That header is exposed as the fields of the Writer and Reader structs. +// +// Strings must be UTF-8 encoded and may only contain Unicode code points +// U+0001 through U+00FF, due to limitations of the GZIP file format. +type Header struct { + Comment string // comment + Extra []byte // "extra data" + ModTime time.Time // modification time + Name string // file name + OS byte // operating system type +} + +// A Reader is an io.Reader that can be read to retrieve +// uncompressed data from a gzip-format compressed file. +// +// In general, a gzip file can be a concatenation of gzip files, +// each with its own header. Reads from the Reader +// return the concatenation of the uncompressed data of each. +// Only the first header is recorded in the Reader fields. +// +// Gzip files store a length and checksum of the uncompressed data. +// The Reader will return a ErrChecksum when Read +// reaches the end of the uncompressed data if it does not +// have the expected length or checksum. Clients should treat data +// returned by Read as tentative until they receive the io.EOF +// marking the end of the data. +type Reader struct { + Header // valid after NewReader or Reader.Reset + r flate.Reader + br *bufio.Reader + decompressor io.ReadCloser + digest uint32 // CRC-32, IEEE polynomial (section 8) + size uint32 // Uncompressed size (section 2.3.1) + buf [512]byte + err error + multistream bool +} + +// NewReader creates a new Reader reading the given reader. +// If r does not also implement io.ByteReader, +// the decompressor may read more data than necessary from r. +// +// It is the caller's responsibility to call Close on the Reader when done. +// +// The Reader.Header fields will be valid in the Reader returned. +func NewReader(r io.Reader) (*Reader, error) { + z := new(Reader) + if err := z.Reset(r); err != nil { + return nil, err + } + return z, nil +} + +// Reset discards the Reader z's state and makes it equivalent to the +// result of its original state from NewReader, but reading from r instead. +// This permits reusing a Reader rather than allocating a new one. +func (z *Reader) Reset(r io.Reader) error { + *z = Reader{ + decompressor: z.decompressor, + multistream: true, + } + if rr, ok := r.(flate.Reader); ok { + z.r = rr + } else { + // Reuse if we can. + if z.br != nil { + z.br.Reset(r) + } else { + z.br = bufio.NewReader(r) + } + z.r = z.br + } + z.Header, z.err = z.readHeader() + return z.err +} + +// Multistream controls whether the reader supports multistream files. +// +// If enabled (the default), the Reader expects the input to be a sequence +// of individually gzipped data streams, each with its own header and +// trailer, ending at EOF. The effect is that the concatenation of a sequence +// of gzipped files is treated as equivalent to the gzip of the concatenation +// of the sequence. This is standard behavior for gzip readers. +// +// Calling Multistream(false) disables this behavior; disabling the behavior +// can be useful when reading file formats that distinguish individual gzip +// data streams or mix gzip data streams with other data streams. +// In this mode, when the Reader reaches the end of the data stream, +// Read returns io.EOF. If the underlying reader implements io.ByteReader, +// it will be left positioned just after the gzip stream. +// To start the next stream, call z.Reset(r) followed by z.Multistream(false). +// If there is no next stream, z.Reset(r) will return io.EOF. +func (z *Reader) Multistream(ok bool) { + z.multistream = ok +} + +// readString reads a NUL-terminated string from z.r. +// It treats the bytes read as being encoded as ISO 8859-1 (Latin-1) and +// will output a string encoded using UTF-8. +// This method always updates z.digest with the data read. +func (z *Reader) readString() (string, error) { + var err error + needConv := false + for i := 0; ; i++ { + if i >= len(z.buf) { + return "", ErrHeader + } + z.buf[i], err = z.r.ReadByte() + if err != nil { + return "", err + } + if z.buf[i] > 0x7f { + needConv = true + } + if z.buf[i] == 0 { + // Digest covers the NUL terminator. + z.digest = crc32.Update(z.digest, crc32.IEEETable, z.buf[:i+1]) + + // Strings are ISO 8859-1, Latin-1 (RFC 1952, section 2.3.1). + if needConv { + s := make([]rune, 0, i) + for _, v := range z.buf[:i] { + s = append(s, rune(v)) + } + return string(s), nil + } + return string(z.buf[:i]), nil + } + } +} + +// readHeader reads the GZIP header according to section 2.3.1. +// This method does not set z.err. +func (z *Reader) readHeader() (hdr Header, err error) { + if _, err = io.ReadFull(z.r, z.buf[:10]); err != nil { + // RFC 1952, section 2.2, says the following: + // A gzip file consists of a series of "members" (compressed data sets). + // + // Other than this, the specification does not clarify whether a + // "series" is defined as "one or more" or "zero or more". To err on the + // side of caution, Go interprets this to mean "zero or more". + // Thus, it is okay to return io.EOF here. + return hdr, err + } + if z.buf[0] != gzipID1 || z.buf[1] != gzipID2 || z.buf[2] != gzipDeflate { + return hdr, ErrHeader + } + flg := z.buf[3] + hdr.ModTime = time.Unix(int64(le.Uint32(z.buf[4:8])), 0) + // z.buf[8] is XFL and is currently ignored. + hdr.OS = z.buf[9] + z.digest = crc32.ChecksumIEEE(z.buf[:10]) + + if flg&flagExtra != 0 { + if _, err = io.ReadFull(z.r, z.buf[:2]); err != nil { + return hdr, noEOF(err) + } + z.digest = crc32.Update(z.digest, crc32.IEEETable, z.buf[:2]) + data := make([]byte, le.Uint16(z.buf[:2])) + if _, err = io.ReadFull(z.r, data); err != nil { + return hdr, noEOF(err) + } + z.digest = crc32.Update(z.digest, crc32.IEEETable, data) + hdr.Extra = data + } + + var s string + if flg&flagName != 0 { + if s, err = z.readString(); err != nil { + return hdr, err + } + hdr.Name = s + } + + if flg&flagComment != 0 { + if s, err = z.readString(); err != nil { + return hdr, err + } + hdr.Comment = s + } + + if flg&flagHdrCrc != 0 { + if _, err = io.ReadFull(z.r, z.buf[:2]); err != nil { + return hdr, noEOF(err) + } + digest := le.Uint16(z.buf[:2]) + if digest != uint16(z.digest) { + return hdr, ErrHeader + } + } + + z.digest = 0 + if z.decompressor == nil { + z.decompressor = flate.NewReader(z.r) + } else { + z.decompressor.(flate.Resetter).Reset(z.r, nil) + } + return hdr, nil +} + +// Read implements io.Reader, reading uncompressed bytes from its underlying Reader. +func (z *Reader) Read(p []byte) (n int, err error) { + if z.err != nil { + return 0, z.err + } + + for n == 0 { + n, z.err = z.decompressor.Read(p) + z.digest = crc32.Update(z.digest, crc32.IEEETable, p[:n]) + z.size += uint32(n) + if z.err != io.EOF { + // In the normal case we return here. + return n, z.err + } + + // Finished file; check checksum and size. + if _, err := io.ReadFull(z.r, z.buf[:8]); err != nil { + z.err = noEOF(err) + return n, z.err + } + digest := le.Uint32(z.buf[:4]) + size := le.Uint32(z.buf[4:8]) + if digest != z.digest || size != z.size { + z.err = ErrChecksum + return n, z.err + } + z.digest, z.size = 0, 0 + + // File is ok; check if there is another. + if !z.multistream { + return n, io.EOF + } + z.err = nil // Remove io.EOF + + if _, z.err = z.readHeader(); z.err != nil { + return n, z.err + } + } + + return n, nil +} + +type crcer interface { + io.Writer + Sum32() uint32 + Reset() +} +type crcUpdater struct { + z *Reader +} + +func (c *crcUpdater) Write(p []byte) (int, error) { + c.z.digest = crc32.Update(c.z.digest, crc32.IEEETable, p) + return len(p), nil +} + +func (c *crcUpdater) Sum32() uint32 { + return c.z.digest +} + +func (c *crcUpdater) Reset() { + c.z.digest = 0 +} + +// WriteTo support the io.WriteTo interface for io.Copy and friends. +func (z *Reader) WriteTo(w io.Writer) (int64, error) { + total := int64(0) + crcWriter := crcer(crc32.NewIEEE()) + if z.digest != 0 { + crcWriter = &crcUpdater{z: z} + } + for { + if z.err != nil { + if z.err == io.EOF { + return total, nil + } + return total, z.err + } + + // We write both to output and digest. + mw := io.MultiWriter(w, crcWriter) + n, err := z.decompressor.(io.WriterTo).WriteTo(mw) + total += n + z.size += uint32(n) + if err != nil { + z.err = err + return total, z.err + } + + // Finished file; check checksum + size. + if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + z.err = err + return total, err + } + z.digest = crcWriter.Sum32() + digest := le.Uint32(z.buf[:4]) + size := le.Uint32(z.buf[4:8]) + if digest != z.digest || size != z.size { + z.err = ErrChecksum + return total, z.err + } + z.digest, z.size = 0, 0 + + // File is ok; check if there is another. + if !z.multistream { + return total, nil + } + crcWriter.Reset() + z.err = nil // Remove io.EOF + + if _, z.err = z.readHeader(); z.err != nil { + if z.err == io.EOF { + return total, nil + } + return total, z.err + } + } +} + +// Close closes the Reader. It does not close the underlying io.Reader. +// In order for the GZIP checksum to be verified, the reader must be +// fully consumed until the io.EOF. +func (z *Reader) Close() error { return z.decompressor.Close() } diff --git a/vendor/github.com/klauspost/compress/gzip/gzip.go b/vendor/github.com/klauspost/compress/gzip/gzip.go new file mode 100644 index 00000000..26203851 --- /dev/null +++ b/vendor/github.com/klauspost/compress/gzip/gzip.go @@ -0,0 +1,269 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gzip + +import ( + "errors" + "fmt" + "hash/crc32" + "io" + + "github.com/klauspost/compress/flate" +) + +// These constants are copied from the flate package, so that code that imports +// "compress/gzip" does not also have to import "compress/flate". +const ( + NoCompression = flate.NoCompression + BestSpeed = flate.BestSpeed + BestCompression = flate.BestCompression + DefaultCompression = flate.DefaultCompression + ConstantCompression = flate.ConstantCompression + HuffmanOnly = flate.HuffmanOnly + + // StatelessCompression will do compression but without maintaining any state + // between Write calls. + // There will be no memory kept between Write calls, + // but compression and speed will be suboptimal. + // Because of this, the size of actual Write calls will affect output size. + StatelessCompression = -3 +) + +// A Writer is an io.WriteCloser. +// Writes to a Writer are compressed and written to w. +type Writer struct { + Header // written at first call to Write, Flush, or Close + w io.Writer + level int + err error + compressor *flate.Writer + digest uint32 // CRC-32, IEEE polynomial (section 8) + size uint32 // Uncompressed size (section 2.3.1) + wroteHeader bool + closed bool + buf [10]byte +} + +// NewWriter returns a new Writer. +// Writes to the returned writer are compressed and written to w. +// +// It is the caller's responsibility to call Close on the WriteCloser when done. +// Writes may be buffered and not flushed until Close. +// +// Callers that wish to set the fields in Writer.Header must do so before +// the first call to Write, Flush, or Close. +func NewWriter(w io.Writer) *Writer { + z, _ := NewWriterLevel(w, DefaultCompression) + return z +} + +// NewWriterLevel is like NewWriter but specifies the compression level instead +// of assuming DefaultCompression. +// +// The compression level can be DefaultCompression, NoCompression, or any +// integer value between BestSpeed and BestCompression inclusive. The error +// returned will be nil if the level is valid. +func NewWriterLevel(w io.Writer, level int) (*Writer, error) { + if level < StatelessCompression || level > BestCompression { + return nil, fmt.Errorf("gzip: invalid compression level: %d", level) + } + z := new(Writer) + z.init(w, level) + return z, nil +} + +func (z *Writer) init(w io.Writer, level int) { + compressor := z.compressor + if level != StatelessCompression { + if compressor != nil { + compressor.Reset(w) + } + } + + *z = Writer{ + Header: Header{ + OS: 255, // unknown + }, + w: w, + level: level, + compressor: compressor, + } +} + +// Reset discards the Writer z's state and makes it equivalent to the +// result of its original state from NewWriter or NewWriterLevel, but +// writing to w instead. This permits reusing a Writer rather than +// allocating a new one. +func (z *Writer) Reset(w io.Writer) { + z.init(w, z.level) +} + +// writeBytes writes a length-prefixed byte slice to z.w. +func (z *Writer) writeBytes(b []byte) error { + if len(b) > 0xffff { + return errors.New("gzip.Write: Extra data is too large") + } + le.PutUint16(z.buf[:2], uint16(len(b))) + _, err := z.w.Write(z.buf[:2]) + if err != nil { + return err + } + _, err = z.w.Write(b) + return err +} + +// writeString writes a UTF-8 string s in GZIP's format to z.w. +// GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1). +func (z *Writer) writeString(s string) (err error) { + // GZIP stores Latin-1 strings; error if non-Latin-1; convert if non-ASCII. + needconv := false + for _, v := range s { + if v == 0 || v > 0xff { + return errors.New("gzip.Write: non-Latin-1 header string") + } + if v > 0x7f { + needconv = true + } + } + if needconv { + b := make([]byte, 0, len(s)) + for _, v := range s { + b = append(b, byte(v)) + } + _, err = z.w.Write(b) + } else { + _, err = io.WriteString(z.w, s) + } + if err != nil { + return err + } + // GZIP strings are NUL-terminated. + z.buf[0] = 0 + _, err = z.w.Write(z.buf[:1]) + return err +} + +// Write writes a compressed form of p to the underlying io.Writer. The +// compressed bytes are not necessarily flushed until the Writer is closed. +func (z *Writer) Write(p []byte) (int, error) { + if z.err != nil { + return 0, z.err + } + var n int + // Write the GZIP header lazily. + if !z.wroteHeader { + z.wroteHeader = true + z.buf[0] = gzipID1 + z.buf[1] = gzipID2 + z.buf[2] = gzipDeflate + z.buf[3] = 0 + if z.Extra != nil { + z.buf[3] |= 0x04 + } + if z.Name != "" { + z.buf[3] |= 0x08 + } + if z.Comment != "" { + z.buf[3] |= 0x10 + } + le.PutUint32(z.buf[4:8], uint32(z.ModTime.Unix())) + if z.level == BestCompression { + z.buf[8] = 2 + } else if z.level == BestSpeed { + z.buf[8] = 4 + } else { + z.buf[8] = 0 + } + z.buf[9] = z.OS + n, z.err = z.w.Write(z.buf[:10]) + if z.err != nil { + return n, z.err + } + if z.Extra != nil { + z.err = z.writeBytes(z.Extra) + if z.err != nil { + return n, z.err + } + } + if z.Name != "" { + z.err = z.writeString(z.Name) + if z.err != nil { + return n, z.err + } + } + if z.Comment != "" { + z.err = z.writeString(z.Comment) + if z.err != nil { + return n, z.err + } + } + + if z.compressor == nil && z.level != StatelessCompression { + z.compressor, _ = flate.NewWriter(z.w, z.level) + } + } + z.size += uint32(len(p)) + z.digest = crc32.Update(z.digest, crc32.IEEETable, p) + if z.level == StatelessCompression { + return len(p), flate.StatelessDeflate(z.w, p, false, nil) + } + n, z.err = z.compressor.Write(p) + return n, z.err +} + +// Flush flushes any pending compressed data to the underlying writer. +// +// It is useful mainly in compressed network protocols, to ensure that +// a remote reader has enough data to reconstruct a packet. Flush does +// not return until the data has been written. If the underlying +// writer returns an error, Flush returns that error. +// +// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH. +func (z *Writer) Flush() error { + if z.err != nil { + return z.err + } + if z.closed || z.level == StatelessCompression { + return nil + } + if !z.wroteHeader { + z.Write(nil) + if z.err != nil { + return z.err + } + } + z.err = z.compressor.Flush() + return z.err +} + +// Close closes the Writer, flushing any unwritten data to the underlying +// io.Writer, but does not close the underlying io.Writer. +func (z *Writer) Close() error { + if z.err != nil { + return z.err + } + if z.closed { + return nil + } + z.closed = true + if !z.wroteHeader { + z.Write(nil) + if z.err != nil { + return z.err + } + } + if z.level == StatelessCompression { + z.err = flate.StatelessDeflate(z.w, nil, true, nil) + } else { + z.err = z.compressor.Close() + } + if z.err != nil { + return z.err + } + le.PutUint32(z.buf[:4], z.digest) + le.PutUint32(z.buf[4:8], z.size) + _, z.err = z.w.Write(z.buf[:8]) + return z.err +} diff --git a/vendor/github.com/klauspost/compress/zip/reader.go b/vendor/github.com/klauspost/compress/zip/reader.go new file mode 100644 index 00000000..65aab5bf --- /dev/null +++ b/vendor/github.com/klauspost/compress/zip/reader.go @@ -0,0 +1,905 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zip + +import ( + "bufio" + "encoding/binary" + "errors" + "hash" + "hash/crc32" + "io" + "io/fs" + "os" + "path" + "sort" + "strings" + "sync" + "time" +) + +var ( + ErrFormat = errors.New("zip: not a valid zip file") + ErrAlgorithm = errors.New("zip: unsupported compression algorithm") + ErrChecksum = errors.New("zip: checksum error") +) + +// A Reader serves content from a ZIP archive. +type Reader struct { + r io.ReaderAt + File []*File + Comment string + decompressors map[uint16]Decompressor + + // Some JAR files are zip files with a prefix that is a bash script. + // The baseOffset field is the start of the zip file proper. + baseOffset int64 + + // fileList is a list of files sorted by ename, + // for use by the Open method. + fileListOnce sync.Once + fileList []fileListEntry +} + +// A ReadCloser is a Reader that must be closed when no longer needed. +type ReadCloser struct { + f *os.File + Reader +} + +// A File is a single file in a ZIP archive. +// The file information is in the embedded FileHeader. +// The file content can be accessed by calling Open. +type File struct { + FileHeader + zip *Reader + zipr io.ReaderAt + headerOffset int64 // includes overall ZIP archive baseOffset + zip64 bool // zip64 extended information extra field presence +} + +// OpenReader will open the Zip file specified by name and return a ReadCloser. +func OpenReader(name string) (*ReadCloser, error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + fi, err := f.Stat() + if err != nil { + f.Close() + return nil, err + } + r := new(ReadCloser) + if err := r.init(f, fi.Size()); err != nil { + f.Close() + return nil, err + } + r.f = f + return r, nil +} + +// NewReader returns a new Reader reading from r, which is assumed to +// have the given size in bytes. +func NewReader(r io.ReaderAt, size int64) (*Reader, error) { + if size < 0 { + return nil, errors.New("zip: size cannot be negative") + } + zr := new(Reader) + if err := zr.init(r, size); err != nil { + return nil, err + } + return zr, nil +} + +func (z *Reader) init(r io.ReaderAt, size int64) error { + end, baseOffset, err := readDirectoryEnd(r, size) + if err != nil { + return err + } + z.r = r + z.baseOffset = baseOffset + // Since the number of directory records is not validated, it is not + // safe to preallocate z.File without first checking that the specified + // number of files is reasonable, since a malformed archive may + // indicate it contains up to 1 << 128 - 1 files. Since each file has a + // header which will be _at least_ 30 bytes we can safely preallocate + // if (data size / 30) >= end.directoryRecords. + if end.directorySize < uint64(size) && (uint64(size)-end.directorySize)/30 >= end.directoryRecords { + z.File = make([]*File, 0, end.directoryRecords) + } + z.Comment = end.comment + rs := io.NewSectionReader(r, 0, size) + if _, err = rs.Seek(z.baseOffset+int64(end.directoryOffset), io.SeekStart); err != nil { + return err + } + buf := bufio.NewReader(rs) + + // The count of files inside a zip is truncated to fit in a uint16. + // Gloss over this by reading headers until we encounter + // a bad one, and then only report an ErrFormat or UnexpectedEOF if + // the file count modulo 65536 is incorrect. + for { + f := &File{zip: z, zipr: r} + err = readDirectoryHeader(f, buf) + + // For compatibility with other zip programs, + // if we have a non-zero base offset and can't read + // the first directory header, try again with a zero + // base offset. + if err == ErrFormat && z.baseOffset != 0 && len(z.File) == 0 { + z.baseOffset = 0 + if _, err = rs.Seek(int64(end.directoryOffset), io.SeekStart); err != nil { + return err + } + buf.Reset(rs) + continue + } + + if err == ErrFormat || err == io.ErrUnexpectedEOF { + break + } + if err != nil { + return err + } + f.headerOffset += z.baseOffset + z.File = append(z.File, f) + } + if uint16(len(z.File)) != uint16(end.directoryRecords) { // only compare 16 bits here + // Return the readDirectoryHeader error if we read + // the wrong number of directory entries. + return err + } + return nil +} + +// RegisterDecompressor registers or overrides a custom decompressor for a +// specific method ID. If a decompressor for a given method is not found, +// Reader will default to looking up the decompressor at the package level. +func (z *Reader) RegisterDecompressor(method uint16, dcomp Decompressor) { + if z.decompressors == nil { + z.decompressors = make(map[uint16]Decompressor) + } + z.decompressors[method] = dcomp +} + +func (z *Reader) decompressor(method uint16) Decompressor { + dcomp := z.decompressors[method] + if dcomp == nil { + dcomp = decompressor(method) + } + return dcomp +} + +// Close closes the Zip file, rendering it unusable for I/O. +func (rc *ReadCloser) Close() error { + return rc.f.Close() +} + +// DataOffset returns the offset of the file's possibly-compressed +// data, relative to the beginning of the zip file. +// +// Most callers should instead use Open, which transparently +// decompresses data and verifies checksums. +func (f *File) DataOffset() (offset int64, err error) { + bodyOffset, err := f.findBodyOffset() + if err != nil { + return + } + return f.headerOffset + bodyOffset, nil +} + +// Open returns a ReadCloser that provides access to the File's contents. +// Multiple files may be read concurrently. +func (f *File) Open() (io.ReadCloser, error) { + bodyOffset, err := f.findBodyOffset() + if err != nil { + return nil, err + } + size := int64(f.CompressedSize64) + r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, size) + dcomp := f.zip.decompressor(f.Method) + if dcomp == nil { + return nil, ErrAlgorithm + } + var rc io.ReadCloser = dcomp(r) + var desr io.Reader + if f.hasDataDescriptor() { + desr = io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset+size, dataDescriptorLen) + } + rc = &checksumReader{ + rc: rc, + hash: crc32.NewIEEE(), + f: f, + desr: desr, + } + return rc, nil +} + +// OpenRaw returns a Reader that provides access to the File's contents without +// decompression. +func (f *File) OpenRaw() (io.Reader, error) { + bodyOffset, err := f.findBodyOffset() + if err != nil { + return nil, err + } + r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, int64(f.CompressedSize64)) + return r, nil +} + +type checksumReader struct { + rc io.ReadCloser + hash hash.Hash32 + nread uint64 // number of bytes read so far + f *File + desr io.Reader // if non-nil, where to read the data descriptor + err error // sticky error +} + +func (r *checksumReader) Stat() (fs.FileInfo, error) { + return headerFileInfo{&r.f.FileHeader}, nil +} + +func (r *checksumReader) Read(b []byte) (n int, err error) { + if r.err != nil { + return 0, r.err + } + n, err = r.rc.Read(b) + r.hash.Write(b[:n]) + r.nread += uint64(n) + if r.nread > r.f.UncompressedSize64 { + return 0, ErrFormat + } + if err == nil { + return + } + if err == io.EOF { + if r.nread != r.f.UncompressedSize64 { + return 0, io.ErrUnexpectedEOF + } + if r.desr != nil { + if err1 := readDataDescriptor(r.desr, r.f); err1 != nil { + if err1 == io.EOF { + err = io.ErrUnexpectedEOF + } else { + err = err1 + } + } else if r.hash.Sum32() != r.f.CRC32 { + err = ErrChecksum + } + } else { + // If there's not a data descriptor, we still compare + // the CRC32 of what we've read against the file header + // or TOC's CRC32, if it seems like it was set. + if r.f.CRC32 != 0 && r.hash.Sum32() != r.f.CRC32 { + err = ErrChecksum + } + } + } + r.err = err + return +} + +func (r *checksumReader) Close() error { return r.rc.Close() } + +// findBodyOffset does the minimum work to verify the file has a header +// and returns the file body offset. +func (f *File) findBodyOffset() (int64, error) { + var buf [fileHeaderLen]byte + if _, err := f.zipr.ReadAt(buf[:], f.headerOffset); err != nil { + return 0, err + } + b := readBuf(buf[:]) + if sig := b.uint32(); sig != fileHeaderSignature { + return 0, ErrFormat + } + b = b[22:] // skip over most of the header + filenameLen := int(b.uint16()) + extraLen := int(b.uint16()) + return int64(fileHeaderLen + filenameLen + extraLen), nil +} + +// readDirectoryHeader attempts to read a directory header from r. +// It returns io.ErrUnexpectedEOF if it cannot read a complete header, +// and ErrFormat if it doesn't find a valid header signature. +func readDirectoryHeader(f *File, r io.Reader) error { + var buf [directoryHeaderLen]byte + if _, err := io.ReadFull(r, buf[:]); err != nil { + return err + } + b := readBuf(buf[:]) + if sig := b.uint32(); sig != directoryHeaderSignature { + return ErrFormat + } + f.CreatorVersion = b.uint16() + f.ReaderVersion = b.uint16() + f.Flags = b.uint16() + f.Method = b.uint16() + f.ModifiedTime = b.uint16() + f.ModifiedDate = b.uint16() + f.CRC32 = b.uint32() + f.CompressedSize = b.uint32() + f.UncompressedSize = b.uint32() + f.CompressedSize64 = uint64(f.CompressedSize) + f.UncompressedSize64 = uint64(f.UncompressedSize) + filenameLen := int(b.uint16()) + extraLen := int(b.uint16()) + commentLen := int(b.uint16()) + b = b[4:] // skipped start disk number and internal attributes (2x uint16) + f.ExternalAttrs = b.uint32() + f.headerOffset = int64(b.uint32()) + d := make([]byte, filenameLen+extraLen+commentLen) + if _, err := io.ReadFull(r, d); err != nil { + return err + } + f.Name = string(d[:filenameLen]) + f.Extra = d[filenameLen : filenameLen+extraLen] + f.Comment = string(d[filenameLen+extraLen:]) + + // Determine the character encoding. + utf8Valid1, utf8Require1 := detectUTF8(f.Name) + utf8Valid2, utf8Require2 := detectUTF8(f.Comment) + switch { + case !utf8Valid1 || !utf8Valid2: + // Name and Comment definitely not UTF-8. + f.NonUTF8 = true + case !utf8Require1 && !utf8Require2: + // Name and Comment use only single-byte runes that overlap with UTF-8. + f.NonUTF8 = false + default: + // Might be UTF-8, might be some other encoding; preserve existing flag. + // Some ZIP writers use UTF-8 encoding without setting the UTF-8 flag. + // Since it is impossible to always distinguish valid UTF-8 from some + // other encoding (e.g., GBK or Shift-JIS), we trust the flag. + f.NonUTF8 = f.Flags&0x800 == 0 + } + + needUSize := f.UncompressedSize == ^uint32(0) + needCSize := f.CompressedSize == ^uint32(0) + needHeaderOffset := f.headerOffset == int64(^uint32(0)) + + // Best effort to find what we need. + // Other zip authors might not even follow the basic format, + // and we'll just ignore the Extra content in that case. + var modified time.Time +parseExtras: + for extra := readBuf(f.Extra); len(extra) >= 4; { // need at least tag and size + fieldTag := extra.uint16() + fieldSize := int(extra.uint16()) + if len(extra) < fieldSize { + break + } + fieldBuf := extra.sub(fieldSize) + + switch fieldTag { + case zip64ExtraID: + f.zip64 = true + + // update directory values from the zip64 extra block. + // They should only be consulted if the sizes read earlier + // are maxed out. + // See golang.org/issue/13367. + if needUSize { + needUSize = false + if len(fieldBuf) < 8 { + return ErrFormat + } + f.UncompressedSize64 = fieldBuf.uint64() + } + if needCSize { + needCSize = false + if len(fieldBuf) < 8 { + return ErrFormat + } + f.CompressedSize64 = fieldBuf.uint64() + } + if needHeaderOffset { + needHeaderOffset = false + if len(fieldBuf) < 8 { + return ErrFormat + } + f.headerOffset = int64(fieldBuf.uint64()) + } + case ntfsExtraID: + if len(fieldBuf) < 4 { + continue parseExtras + } + fieldBuf.uint32() // reserved (ignored) + for len(fieldBuf) >= 4 { // need at least tag and size + attrTag := fieldBuf.uint16() + attrSize := int(fieldBuf.uint16()) + if len(fieldBuf) < attrSize { + continue parseExtras + } + attrBuf := fieldBuf.sub(attrSize) + if attrTag != 1 || attrSize != 24 { + continue // Ignore irrelevant attributes + } + + const ticksPerSecond = 1e7 // Windows timestamp resolution + ts := int64(attrBuf.uint64()) // ModTime since Windows epoch + secs := int64(ts / ticksPerSecond) + nsecs := (1e9 / ticksPerSecond) * int64(ts%ticksPerSecond) + epoch := time.Date(1601, time.January, 1, 0, 0, 0, 0, time.UTC) + modified = time.Unix(epoch.Unix()+secs, nsecs) + } + case unixExtraID, infoZipUnixExtraID: + if len(fieldBuf) < 8 { + continue parseExtras + } + fieldBuf.uint32() // AcTime (ignored) + ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch + modified = time.Unix(ts, 0) + case extTimeExtraID: + if len(fieldBuf) < 5 || fieldBuf.uint8()&1 == 0 { + continue parseExtras + } + ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch + modified = time.Unix(ts, 0) + } + } + + msdosModified := msDosTimeToTime(f.ModifiedDate, f.ModifiedTime) + f.Modified = msdosModified + if !modified.IsZero() { + f.Modified = modified.UTC() + + // If legacy MS-DOS timestamps are set, we can use the delta between + // the legacy and extended versions to estimate timezone offset. + // + // A non-UTC timezone is always used (even if offset is zero). + // Thus, FileHeader.Modified.Location() == time.UTC is useful for + // determining whether extended timestamps are present. + // This is necessary for users that need to do additional time + // calculations when dealing with legacy ZIP formats. + if f.ModifiedTime != 0 || f.ModifiedDate != 0 { + f.Modified = modified.In(timeZone(msdosModified.Sub(modified))) + } + } + + // Assume that uncompressed size 2³²-1 could plausibly happen in + // an old zip32 file that was sharding inputs into the largest chunks + // possible (or is just malicious; search the web for 42.zip). + // If needUSize is true still, it means we didn't see a zip64 extension. + // As long as the compressed size is not also 2³²-1 (implausible) + // and the header is not also 2³²-1 (equally implausible), + // accept the uncompressed size 2³²-1 as valid. + // If nothing else, this keeps archive/zip working with 42.zip. + _ = needUSize + + if needCSize || needHeaderOffset { + return ErrFormat + } + + return nil +} + +func readDataDescriptor(r io.Reader, f *File) error { + var buf [dataDescriptorLen]byte + // The spec says: "Although not originally assigned a + // signature, the value 0x08074b50 has commonly been adopted + // as a signature value for the data descriptor record. + // Implementers should be aware that ZIP files may be + // encountered with or without this signature marking data + // descriptors and should account for either case when reading + // ZIP files to ensure compatibility." + // + // dataDescriptorLen includes the size of the signature but + // first read just those 4 bytes to see if it exists. + if _, err := io.ReadFull(r, buf[:4]); err != nil { + return err + } + off := 0 + maybeSig := readBuf(buf[:4]) + if maybeSig.uint32() != dataDescriptorSignature { + // No data descriptor signature. Keep these four + // bytes. + off += 4 + } + if _, err := io.ReadFull(r, buf[off:12]); err != nil { + return err + } + b := readBuf(buf[:12]) + if b.uint32() != f.CRC32 { + return ErrChecksum + } + + // The two sizes that follow here can be either 32 bits or 64 bits + // but the spec is not very clear on this and different + // interpretations has been made causing incompatibilities. We + // already have the sizes from the central directory so we can + // just ignore these. + + return nil +} + +func readDirectoryEnd(r io.ReaderAt, size int64) (dir *directoryEnd, baseOffset int64, err error) { + // look for directoryEndSignature in the last 1k, then in the last 65k + var buf []byte + var directoryEndOffset int64 + for i, bLen := range []int64{1024, 65 * 1024} { + if bLen > size { + bLen = size + } + buf = make([]byte, int(bLen)) + if _, err := r.ReadAt(buf, size-bLen); err != nil && err != io.EOF { + return nil, 0, err + } + if p := findSignatureInBlock(buf); p >= 0 { + buf = buf[p:] + directoryEndOffset = size - bLen + int64(p) + break + } + if i == 1 || bLen == size { + return nil, 0, ErrFormat + } + } + + // read header into struct + b := readBuf(buf[4:]) // skip signature + d := &directoryEnd{ + diskNbr: uint32(b.uint16()), + dirDiskNbr: uint32(b.uint16()), + dirRecordsThisDisk: uint64(b.uint16()), + directoryRecords: uint64(b.uint16()), + directorySize: uint64(b.uint32()), + directoryOffset: uint64(b.uint32()), + commentLen: b.uint16(), + } + l := int(d.commentLen) + if l > len(b) { + return nil, 0, errors.New("zip: invalid comment length") + } + d.comment = string(b[:l]) + + // These values mean that the file can be a zip64 file + if d.directoryRecords == 0xffff || d.directorySize == 0xffff || d.directoryOffset == 0xffffffff { + p, err := findDirectory64End(r, directoryEndOffset) + if err == nil && p >= 0 { + directoryEndOffset = p + err = readDirectory64End(r, p, d) + } + if err != nil { + return nil, 0, err + } + } + + baseOffset = directoryEndOffset - int64(d.directorySize) - int64(d.directoryOffset) + + // Make sure directoryOffset points to somewhere in our file. + if o := baseOffset + int64(d.directoryOffset); o < 0 || o >= size { + return nil, 0, ErrFormat + } + return d, baseOffset, nil +} + +// findDirectory64End tries to read the zip64 locator just before the +// directory end and returns the offset of the zip64 directory end if +// found. +func findDirectory64End(r io.ReaderAt, directoryEndOffset int64) (int64, error) { + locOffset := directoryEndOffset - directory64LocLen + if locOffset < 0 { + return -1, nil // no need to look for a header outside the file + } + buf := make([]byte, directory64LocLen) + if _, err := r.ReadAt(buf, locOffset); err != nil { + return -1, err + } + b := readBuf(buf) + if sig := b.uint32(); sig != directory64LocSignature { + return -1, nil + } + if b.uint32() != 0 { // number of the disk with the start of the zip64 end of central directory + return -1, nil // the file is not a valid zip64-file + } + p := b.uint64() // relative offset of the zip64 end of central directory record + if b.uint32() != 1 { // total number of disks + return -1, nil // the file is not a valid zip64-file + } + return int64(p), nil +} + +// readDirectory64End reads the zip64 directory end and updates the +// directory end with the zip64 directory end values. +func readDirectory64End(r io.ReaderAt, offset int64, d *directoryEnd) (err error) { + buf := make([]byte, directory64EndLen) + if _, err := r.ReadAt(buf, offset); err != nil { + return err + } + + b := readBuf(buf) + if sig := b.uint32(); sig != directory64EndSignature { + return ErrFormat + } + + b = b[12:] // skip dir size, version and version needed (uint64 + 2x uint16) + d.diskNbr = b.uint32() // number of this disk + d.dirDiskNbr = b.uint32() // number of the disk with the start of the central directory + d.dirRecordsThisDisk = b.uint64() // total number of entries in the central directory on this disk + d.directoryRecords = b.uint64() // total number of entries in the central directory + d.directorySize = b.uint64() // size of the central directory + d.directoryOffset = b.uint64() // offset of start of central directory with respect to the starting disk number + + return nil +} + +func findSignatureInBlock(b []byte) int { + for i := len(b) - directoryEndLen; i >= 0; i-- { + // defined from directoryEndSignature in struct.go + if b[i] == 'P' && b[i+1] == 'K' && b[i+2] == 0x05 && b[i+3] == 0x06 { + // n is length of comment + n := int(b[i+directoryEndLen-2]) | int(b[i+directoryEndLen-1])<<8 + if n+directoryEndLen+i <= len(b) { + return i + } + } + } + return -1 +} + +type readBuf []byte + +func (b *readBuf) uint8() uint8 { + v := (*b)[0] + *b = (*b)[1:] + return v +} + +func (b *readBuf) uint16() uint16 { + v := binary.LittleEndian.Uint16(*b) + *b = (*b)[2:] + return v +} + +func (b *readBuf) uint32() uint32 { + v := binary.LittleEndian.Uint32(*b) + *b = (*b)[4:] + return v +} + +func (b *readBuf) uint64() uint64 { + v := binary.LittleEndian.Uint64(*b) + *b = (*b)[8:] + return v +} + +func (b *readBuf) sub(n int) readBuf { + b2 := (*b)[:n] + *b = (*b)[n:] + return b2 +} + +// A fileListEntry is a File and its ename. +// If file == nil, the fileListEntry describes a directory without metadata. +type fileListEntry struct { + name string + file *File + isDir bool + isDup bool +} + +type fileInfoDirEntry interface { + fs.FileInfo + fs.DirEntry +} + +func (e *fileListEntry) stat() (fileInfoDirEntry, error) { + if e.isDup { + return nil, errors.New(e.name + ": duplicate entries in zip file") + } + if !e.isDir { + return headerFileInfo{&e.file.FileHeader}, nil + } + return e, nil +} + +// Only used for directories. +func (f *fileListEntry) Name() string { _, elem, _ := split(f.name); return elem } +func (f *fileListEntry) Size() int64 { return 0 } +func (f *fileListEntry) Mode() fs.FileMode { return fs.ModeDir | 0555 } +func (f *fileListEntry) Type() fs.FileMode { return fs.ModeDir } +func (f *fileListEntry) IsDir() bool { return true } +func (f *fileListEntry) Sys() interface{} { return nil } + +func (f *fileListEntry) ModTime() time.Time { + if f.file == nil { + return time.Time{} + } + return f.file.FileHeader.Modified.UTC() +} + +func (f *fileListEntry) Info() (fs.FileInfo, error) { return f, nil } + +// toValidName coerces name to be a valid name for fs.FS.Open. +func toValidName(name string) string { + name = strings.ReplaceAll(name, `\`, `/`) + p := path.Clean(name) + if strings.HasPrefix(p, "/") { + p = p[len("/"):] + } + for strings.HasPrefix(p, "../") { + p = p[len("../"):] + } + return p +} + +func (r *Reader) initFileList() { + r.fileListOnce.Do(func() { + // files and knownDirs map from a file/directory name + // to an index into the r.fileList entry that we are + // building. They are used to mark duplicate entries. + files := make(map[string]int) + knownDirs := make(map[string]int) + + // dirs[name] is true if name is known to be a directory, + // because it appears as a prefix in a path. + dirs := make(map[string]bool) + + for _, file := range r.File { + isDir := len(file.Name) > 0 && file.Name[len(file.Name)-1] == '/' + name := toValidName(file.Name) + if name == "" { + continue + } + + if idx, ok := files[name]; ok { + r.fileList[idx].isDup = true + continue + } + if idx, ok := knownDirs[name]; ok { + r.fileList[idx].isDup = true + continue + } + + for dir := path.Dir(name); dir != "."; dir = path.Dir(dir) { + dirs[dir] = true + } + + idx := len(r.fileList) + entry := fileListEntry{ + name: name, + file: file, + isDir: isDir, + } + r.fileList = append(r.fileList, entry) + if isDir { + knownDirs[name] = idx + } else { + files[name] = idx + } + } + for dir := range dirs { + if _, ok := knownDirs[dir]; !ok { + if idx, ok := files[dir]; ok { + r.fileList[idx].isDup = true + } else { + entry := fileListEntry{ + name: dir, + file: nil, + isDir: true, + } + r.fileList = append(r.fileList, entry) + } + } + } + + sort.Slice(r.fileList, func(i, j int) bool { return fileEntryLess(r.fileList[i].name, r.fileList[j].name) }) + }) +} + +func fileEntryLess(x, y string) bool { + xdir, xelem, _ := split(x) + ydir, yelem, _ := split(y) + return xdir < ydir || xdir == ydir && xelem < yelem +} + +// Open opens the named file in the ZIP archive, +// using the semantics of fs.FS.Open: +// paths are always slash separated, with no +// leading / or ../ elements. +func (r *Reader) Open(name string) (fs.File, error) { + r.initFileList() + + if !fs.ValidPath(name) { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid} + } + e := r.openLookup(name) + if e == nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} + } + if e.isDir { + return &openDir{e, r.openReadDir(name), 0}, nil + } + rc, err := e.file.Open() + if err != nil { + return nil, err + } + return rc.(fs.File), nil +} + +func split(name string) (dir, elem string, isDir bool) { + if len(name) > 0 && name[len(name)-1] == '/' { + isDir = true + name = name[:len(name)-1] + } + i := len(name) - 1 + for i >= 0 && name[i] != '/' { + i-- + } + if i < 0 { + return ".", name, isDir + } + return name[:i], name[i+1:], isDir +} + +var dotFile = &fileListEntry{name: "./", isDir: true} + +func (r *Reader) openLookup(name string) *fileListEntry { + if name == "." { + return dotFile + } + + dir, elem, _ := split(name) + files := r.fileList + i := sort.Search(len(files), func(i int) bool { + idir, ielem, _ := split(files[i].name) + return idir > dir || idir == dir && ielem >= elem + }) + if i < len(files) { + fname := files[i].name + if fname == name || len(fname) == len(name)+1 && fname[len(name)] == '/' && fname[:len(name)] == name { + return &files[i] + } + } + return nil +} + +func (r *Reader) openReadDir(dir string) []fileListEntry { + files := r.fileList + i := sort.Search(len(files), func(i int) bool { + idir, _, _ := split(files[i].name) + return idir >= dir + }) + j := sort.Search(len(files), func(j int) bool { + jdir, _, _ := split(files[j].name) + return jdir > dir + }) + return files[i:j] +} + +type openDir struct { + e *fileListEntry + files []fileListEntry + offset int +} + +func (d *openDir) Close() error { return nil } +func (d *openDir) Stat() (fs.FileInfo, error) { return d.e.stat() } + +func (d *openDir) Read([]byte) (int, error) { + return 0, &fs.PathError{Op: "read", Path: d.e.name, Err: errors.New("is a directory")} +} + +func (d *openDir) ReadDir(count int) ([]fs.DirEntry, error) { + n := len(d.files) - d.offset + if count > 0 && n > count { + n = count + } + if n == 0 { + if count <= 0 { + return nil, nil + } + return nil, io.EOF + } + list := make([]fs.DirEntry, n) + for i := range list { + s, err := d.files[d.offset+i].stat() + if err != nil { + return nil, err + } + list[i] = s + } + d.offset += n + return list, nil +} diff --git a/vendor/github.com/klauspost/compress/zip/register.go b/vendor/github.com/klauspost/compress/zip/register.go new file mode 100644 index 00000000..ca8c13ce --- /dev/null +++ b/vendor/github.com/klauspost/compress/zip/register.go @@ -0,0 +1,148 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zip + +import ( + "errors" + "io" + "sync" + + "github.com/klauspost/compress/flate" +) + +// A Compressor returns a new compressing writer, writing to w. +// The WriteCloser's Close method must be used to flush pending data to w. +// The Compressor itself must be safe to invoke from multiple goroutines +// simultaneously, but each returned writer will be used only by +// one goroutine at a time. +type Compressor func(w io.Writer) (io.WriteCloser, error) + +// A Decompressor returns a new decompressing reader, reading from r. +// The ReadCloser's Close method must be used to release associated resources. +// The Decompressor itself must be safe to invoke from multiple goroutines +// simultaneously, but each returned reader will be used only by +// one goroutine at a time. +type Decompressor func(r io.Reader) io.ReadCloser + +var flateWriterPool sync.Pool + +func newFlateWriter(w io.Writer) io.WriteCloser { + fw, ok := flateWriterPool.Get().(*flate.Writer) + if ok { + fw.Reset(w) + } else { + fw, _ = flate.NewWriter(w, 5) + } + return &pooledFlateWriter{fw: fw} +} + +type pooledFlateWriter struct { + mu sync.Mutex // guards Close and Write + fw *flate.Writer +} + +func (w *pooledFlateWriter) Write(p []byte) (n int, err error) { + w.mu.Lock() + defer w.mu.Unlock() + if w.fw == nil { + return 0, errors.New("Write after Close") + } + return w.fw.Write(p) +} + +func (w *pooledFlateWriter) Close() error { + w.mu.Lock() + defer w.mu.Unlock() + var err error + if w.fw != nil { + err = w.fw.Close() + flateWriterPool.Put(w.fw) + w.fw = nil + } + return err +} + +var flateReaderPool sync.Pool + +func newFlateReader(r io.Reader) io.ReadCloser { + fr, ok := flateReaderPool.Get().(io.ReadCloser) + if ok { + fr.(flate.Resetter).Reset(r, nil) + } else { + fr = flate.NewReader(r) + } + return &pooledFlateReader{fr: fr} +} + +type pooledFlateReader struct { + mu sync.Mutex // guards Close and Read + fr io.ReadCloser +} + +func (r *pooledFlateReader) Read(p []byte) (n int, err error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.fr == nil { + return 0, errors.New("Read after Close") + } + return r.fr.Read(p) +} + +func (r *pooledFlateReader) Close() error { + r.mu.Lock() + defer r.mu.Unlock() + var err error + if r.fr != nil { + err = r.fr.Close() + flateReaderPool.Put(r.fr) + r.fr = nil + } + return err +} + +var ( + compressors sync.Map // map[uint16]Compressor + decompressors sync.Map // map[uint16]Decompressor +) + +func init() { + compressors.Store(Store, Compressor(func(w io.Writer) (io.WriteCloser, error) { return &nopCloser{w}, nil })) + compressors.Store(Deflate, Compressor(func(w io.Writer) (io.WriteCloser, error) { return newFlateWriter(w), nil })) + + decompressors.Store(Store, Decompressor(io.NopCloser)) + decompressors.Store(Deflate, Decompressor(newFlateReader)) +} + +// RegisterDecompressor allows custom decompressors for a specified method ID. +// The common methods Store and Deflate are built in. +func RegisterDecompressor(method uint16, dcomp Decompressor) { + if _, dup := decompressors.LoadOrStore(method, dcomp); dup { + panic("decompressor already registered") + } +} + +// RegisterCompressor registers custom compressors for a specified method ID. +// The common methods Store and Deflate are built in. +func RegisterCompressor(method uint16, comp Compressor) { + if _, dup := compressors.LoadOrStore(method, comp); dup { + panic("compressor already registered") + } +} + +func compressor(method uint16) Compressor { + ci, ok := compressors.Load(method) + if !ok { + return nil + } + return ci.(Compressor) +} + +func decompressor(method uint16) Decompressor { + di, ok := decompressors.Load(method) + if !ok { + return nil + } + return di.(Decompressor) +} diff --git a/vendor/github.com/klauspost/compress/zip/struct.go b/vendor/github.com/klauspost/compress/zip/struct.go new file mode 100644 index 00000000..88effedc --- /dev/null +++ b/vendor/github.com/klauspost/compress/zip/struct.go @@ -0,0 +1,392 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package zip provides support for reading and writing ZIP archives. + +See: https://www.pkware.com/appnote + +This package does not support disk spanning. + +A note about ZIP64: + +To be backwards compatible the FileHeader has both 32 and 64 bit Size +fields. The 64 bit fields will always contain the correct value and +for normal archives both fields will be the same. For files requiring +the ZIP64 format the 32 bit fields will be 0xffffffff and the 64 bit +fields must be used instead. +*/ +package zip + +import ( + "io/fs" + "path" + "time" +) + +// Compression methods. +const ( + Store uint16 = 0 // no compression + Deflate uint16 = 8 // DEFLATE compressed +) + +const ( + fileHeaderSignature = 0x04034b50 + directoryHeaderSignature = 0x02014b50 + directoryEndSignature = 0x06054b50 + directory64LocSignature = 0x07064b50 + directory64EndSignature = 0x06064b50 + dataDescriptorSignature = 0x08074b50 // de-facto standard; required by OS X Finder + fileHeaderLen = 30 // + filename + extra + directoryHeaderLen = 46 // + filename + extra + comment + directoryEndLen = 22 // + comment + dataDescriptorLen = 16 // four uint32: descriptor signature, crc32, compressed size, size + dataDescriptor64Len = 24 // two uint32: signature, crc32 | two uint64: compressed size, size + directory64LocLen = 20 // + directory64EndLen = 56 // + extra + + // Constants for the first byte in CreatorVersion. + creatorFAT = 0 + creatorUnix = 3 + creatorNTFS = 11 + creatorVFAT = 14 + creatorMacOSX = 19 + + // Version numbers. + zipVersion20 = 20 // 2.0 + zipVersion45 = 45 // 4.5 (reads and writes zip64 archives) + + // Limits for non zip64 files. + uint16max = (1 << 16) - 1 + uint32max = (1 << 32) - 1 + + // Extra header IDs. + // + // IDs 0..31 are reserved for official use by PKWARE. + // IDs above that range are defined by third-party vendors. + // Since ZIP lacked high precision timestamps (nor a official specification + // of the timezone used for the date fields), many competing extra fields + // have been invented. Pervasive use effectively makes them "official". + // + // See http://mdfs.net/Docs/Comp/Archiving/Zip/ExtraField + zip64ExtraID = 0x0001 // Zip64 extended information + ntfsExtraID = 0x000a // NTFS + unixExtraID = 0x000d // UNIX + extTimeExtraID = 0x5455 // Extended timestamp + infoZipUnixExtraID = 0x5855 // Info-ZIP Unix extension +) + +// FileHeader describes a file within a zip file. +// See the zip spec for details. +type FileHeader struct { + // Name is the name of the file. + // + // It must be a relative path, not start with a drive letter (such as "C:"), + // and must use forward slashes instead of back slashes. A trailing slash + // indicates that this file is a directory and should have no data. + // + // When reading zip files, the Name field is populated from + // the zip file directly and is not validated for correctness. + // It is the caller's responsibility to sanitize it as + // appropriate, including canonicalizing slash directions, + // validating that paths are relative, and preventing path + // traversal through filenames ("../../../"). + Name string + + // Comment is any arbitrary user-defined string shorter than 64KiB. + Comment string + + // NonUTF8 indicates that Name and Comment are not encoded in UTF-8. + // + // By specification, the only other encoding permitted should be CP-437, + // but historically many ZIP readers interpret Name and Comment as whatever + // the system's local character encoding happens to be. + // + // This flag should only be set if the user intends to encode a non-portable + // ZIP file for a specific localized region. Otherwise, the Writer + // automatically sets the ZIP format's UTF-8 flag for valid UTF-8 strings. + NonUTF8 bool + + CreatorVersion uint16 + ReaderVersion uint16 + Flags uint16 + + // Method is the compression method. If zero, Store is used. + Method uint16 + + // Modified is the modified time of the file. + // + // When reading, an extended timestamp is preferred over the legacy MS-DOS + // date field, and the offset between the times is used as the timezone. + // If only the MS-DOS date is present, the timezone is assumed to be UTC. + // + // When writing, an extended timestamp (which is timezone-agnostic) is + // always emitted. The legacy MS-DOS date field is encoded according to the + // location of the Modified time. + Modified time.Time + ModifiedTime uint16 // Deprecated: Legacy MS-DOS date; use Modified instead. + ModifiedDate uint16 // Deprecated: Legacy MS-DOS time; use Modified instead. + + CRC32 uint32 + CompressedSize uint32 // Deprecated: Use CompressedSize64 instead. + UncompressedSize uint32 // Deprecated: Use UncompressedSize64 instead. + CompressedSize64 uint64 + UncompressedSize64 uint64 + Extra []byte + ExternalAttrs uint32 // Meaning depends on CreatorVersion +} + +// FileInfo returns an fs.FileInfo for the FileHeader. +func (h *FileHeader) FileInfo() fs.FileInfo { + return headerFileInfo{h} +} + +// headerFileInfo implements fs.FileInfo. +type headerFileInfo struct { + fh *FileHeader +} + +func (fi headerFileInfo) Name() string { return path.Base(fi.fh.Name) } +func (fi headerFileInfo) Size() int64 { + if fi.fh.UncompressedSize64 > 0 { + return int64(fi.fh.UncompressedSize64) + } + return int64(fi.fh.UncompressedSize) +} +func (fi headerFileInfo) IsDir() bool { return fi.Mode().IsDir() } +func (fi headerFileInfo) ModTime() time.Time { + if fi.fh.Modified.IsZero() { + return fi.fh.ModTime() + } + return fi.fh.Modified.UTC() +} +func (fi headerFileInfo) Mode() fs.FileMode { return fi.fh.Mode() } +func (fi headerFileInfo) Type() fs.FileMode { return fi.fh.Mode().Type() } +func (fi headerFileInfo) Sys() interface{} { return fi.fh } + +func (fi headerFileInfo) Info() (fs.FileInfo, error) { return fi, nil } + +// FileInfoHeader creates a partially-populated FileHeader from an +// fs.FileInfo. +// Because fs.FileInfo's Name method returns only the base name of +// the file it describes, it may be necessary to modify the Name field +// of the returned header to provide the full path name of the file. +// If compression is desired, callers should set the FileHeader.Method +// field; it is unset by default. +func FileInfoHeader(fi fs.FileInfo) (*FileHeader, error) { + size := fi.Size() + fh := &FileHeader{ + Name: fi.Name(), + UncompressedSize64: uint64(size), + } + fh.SetModTime(fi.ModTime()) + fh.SetMode(fi.Mode()) + if fh.UncompressedSize64 > uint32max { + fh.UncompressedSize = uint32max + } else { + fh.UncompressedSize = uint32(fh.UncompressedSize64) + } + return fh, nil +} + +type directoryEnd struct { + diskNbr uint32 // unused + dirDiskNbr uint32 // unused + dirRecordsThisDisk uint64 // unused + directoryRecords uint64 + directorySize uint64 + directoryOffset uint64 // relative to file + commentLen uint16 + comment string +} + +// timeZone returns a *time.Location based on the provided offset. +// If the offset is non-sensible, then this uses an offset of zero. +func timeZone(offset time.Duration) *time.Location { + const ( + minOffset = -12 * time.Hour // E.g., Baker island at -12:00 + maxOffset = +14 * time.Hour // E.g., Line island at +14:00 + offsetAlias = 15 * time.Minute // E.g., Nepal at +5:45 + ) + offset = offset.Round(offsetAlias) + if offset < minOffset || maxOffset < offset { + offset = 0 + } + return time.FixedZone("", int(offset/time.Second)) +} + +// msDosTimeToTime converts an MS-DOS date and time into a time.Time. +// The resolution is 2s. +// See: https://msdn.microsoft.com/en-us/library/ms724247(v=VS.85).aspx +func msDosTimeToTime(dosDate, dosTime uint16) time.Time { + return time.Date( + // date bits 0-4: day of month; 5-8: month; 9-15: years since 1980 + int(dosDate>>9+1980), + time.Month(dosDate>>5&0xf), + int(dosDate&0x1f), + + // time bits 0-4: second/2; 5-10: minute; 11-15: hour + int(dosTime>>11), + int(dosTime>>5&0x3f), + int(dosTime&0x1f*2), + 0, // nanoseconds + + time.UTC, + ) +} + +// timeToMsDosTime converts a time.Time to an MS-DOS date and time. +// The resolution is 2s. +// See: https://msdn.microsoft.com/en-us/library/ms724274(v=VS.85).aspx +func timeToMsDosTime(t time.Time) (fDate uint16, fTime uint16) { + fDate = uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9) + fTime = uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11) + return +} + +// ModTime returns the modification time in UTC using the legacy +// ModifiedDate and ModifiedTime fields. +// +// Deprecated: Use Modified instead. +func (h *FileHeader) ModTime() time.Time { + return msDosTimeToTime(h.ModifiedDate, h.ModifiedTime) +} + +// SetModTime sets the Modified, ModifiedTime, and ModifiedDate fields +// to the given time in UTC. +// +// Deprecated: Use Modified instead. +func (h *FileHeader) SetModTime(t time.Time) { + t = t.UTC() // Convert to UTC for compatibility + h.Modified = t + h.ModifiedDate, h.ModifiedTime = timeToMsDosTime(t) +} + +const ( + // Unix constants. The specification doesn't mention them, + // but these seem to be the values agreed on by tools. + s_IFMT = 0xf000 + s_IFSOCK = 0xc000 + s_IFLNK = 0xa000 + s_IFREG = 0x8000 + s_IFBLK = 0x6000 + s_IFDIR = 0x4000 + s_IFCHR = 0x2000 + s_IFIFO = 0x1000 + s_ISUID = 0x800 + s_ISGID = 0x400 + s_ISVTX = 0x200 + + msdosDir = 0x10 + msdosReadOnly = 0x01 +) + +// Mode returns the permission and mode bits for the FileHeader. +func (h *FileHeader) Mode() (mode fs.FileMode) { + switch h.CreatorVersion >> 8 { + case creatorUnix, creatorMacOSX: + mode = unixModeToFileMode(h.ExternalAttrs >> 16) + case creatorNTFS, creatorVFAT, creatorFAT: + mode = msdosModeToFileMode(h.ExternalAttrs) + } + if len(h.Name) > 0 && h.Name[len(h.Name)-1] == '/' { + mode |= fs.ModeDir + } + return mode +} + +// SetMode changes the permission and mode bits for the FileHeader. +func (h *FileHeader) SetMode(mode fs.FileMode) { + h.CreatorVersion = h.CreatorVersion&0xff | creatorUnix<<8 + h.ExternalAttrs = fileModeToUnixMode(mode) << 16 + + // set MSDOS attributes too, as the original zip does. + if mode&fs.ModeDir != 0 { + h.ExternalAttrs |= msdosDir + } + if mode&0200 == 0 { + h.ExternalAttrs |= msdosReadOnly + } +} + +// isZip64 reports whether the file size exceeds the 32 bit limit +func (h *FileHeader) isZip64() bool { + return h.CompressedSize64 >= uint32max || h.UncompressedSize64 >= uint32max +} + +func (f *FileHeader) hasDataDescriptor() bool { + return f.Flags&0x8 != 0 +} + +func msdosModeToFileMode(m uint32) (mode fs.FileMode) { + if m&msdosDir != 0 { + mode = fs.ModeDir | 0777 + } else { + mode = 0666 + } + if m&msdosReadOnly != 0 { + mode &^= 0222 + } + return mode +} + +func fileModeToUnixMode(mode fs.FileMode) uint32 { + var m uint32 + switch mode & fs.ModeType { + default: + m = s_IFREG + case fs.ModeDir: + m = s_IFDIR + case fs.ModeSymlink: + m = s_IFLNK + case fs.ModeNamedPipe: + m = s_IFIFO + case fs.ModeSocket: + m = s_IFSOCK + case fs.ModeDevice: + m = s_IFBLK + case fs.ModeDevice | fs.ModeCharDevice: + m = s_IFCHR + } + if mode&fs.ModeSetuid != 0 { + m |= s_ISUID + } + if mode&fs.ModeSetgid != 0 { + m |= s_ISGID + } + if mode&fs.ModeSticky != 0 { + m |= s_ISVTX + } + return m | uint32(mode&0777) +} + +func unixModeToFileMode(m uint32) fs.FileMode { + mode := fs.FileMode(m & 0777) + switch m & s_IFMT { + case s_IFBLK: + mode |= fs.ModeDevice + case s_IFCHR: + mode |= fs.ModeDevice | fs.ModeCharDevice + case s_IFDIR: + mode |= fs.ModeDir + case s_IFIFO: + mode |= fs.ModeNamedPipe + case s_IFLNK: + mode |= fs.ModeSymlink + case s_IFREG: + // nothing to do + case s_IFSOCK: + mode |= fs.ModeSocket + } + if m&s_ISGID != 0 { + mode |= fs.ModeSetgid + } + if m&s_ISUID != 0 { + mode |= fs.ModeSetuid + } + if m&s_ISVTX != 0 { + mode |= fs.ModeSticky + } + return mode +} diff --git a/vendor/github.com/klauspost/compress/zip/writer.go b/vendor/github.com/klauspost/compress/zip/writer.go new file mode 100644 index 00000000..0eb03782 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zip/writer.go @@ -0,0 +1,640 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zip + +import ( + "bufio" + "encoding/binary" + "errors" + "hash" + "hash/crc32" + "io" + "strings" + "unicode/utf8" +) + +var ( + errLongName = errors.New("zip: FileHeader.Name too long") + errLongExtra = errors.New("zip: FileHeader.Extra too long") +) + +// Writer implements a zip file writer. +type Writer struct { + cw *countWriter + dir []*header + last *fileWriter + closed bool + compressors map[uint16]Compressor + comment string + + // testHookCloseSizeOffset if non-nil is called with the size + // of offset of the central directory at Close. + testHookCloseSizeOffset func(size, offset uint64) +} + +type header struct { + *FileHeader + offset uint64 + raw bool +} + +// NewWriter returns a new Writer writing a zip file to w. +func NewWriter(w io.Writer) *Writer { + return &Writer{cw: &countWriter{w: bufio.NewWriter(w)}} +} + +// SetOffset sets the offset of the beginning of the zip data within the +// underlying writer. It should be used when the zip data is appended to an +// existing file, such as a binary executable. +// It must be called before any data is written. +func (w *Writer) SetOffset(n int64) { + if w.cw.count != 0 { + panic("zip: SetOffset called after data was written") + } + w.cw.count = n +} + +// Flush flushes any buffered data to the underlying writer. +// Calling Flush is not normally necessary; calling Close is sufficient. +func (w *Writer) Flush() error { + return w.cw.w.(*bufio.Writer).Flush() +} + +// SetComment sets the end-of-central-directory comment field. +// It can only be called before Close. +func (w *Writer) SetComment(comment string) error { + if len(comment) > uint16max { + return errors.New("zip: Writer.Comment too long") + } + w.comment = comment + return nil +} + +// Close finishes writing the zip file by writing the central directory. +// It does not close the underlying writer. +func (w *Writer) Close() error { + if w.last != nil && !w.last.closed { + if err := w.last.close(); err != nil { + return err + } + w.last = nil + } + if w.closed { + return errors.New("zip: writer closed twice") + } + w.closed = true + + // write central directory + start := w.cw.count + for _, h := range w.dir { + var buf [directoryHeaderLen]byte + b := writeBuf(buf[:]) + b.uint32(uint32(directoryHeaderSignature)) + b.uint16(h.CreatorVersion) + b.uint16(h.ReaderVersion) + b.uint16(h.Flags) + b.uint16(h.Method) + b.uint16(h.ModifiedTime) + b.uint16(h.ModifiedDate) + b.uint32(h.CRC32) + if h.isZip64() || h.offset >= uint32max { + // the file needs a zip64 header. store maxint in both + // 32 bit size fields (and offset later) to signal that the + // zip64 extra header should be used. + b.uint32(uint32max) // compressed size + b.uint32(uint32max) // uncompressed size + + // append a zip64 extra block to Extra + var buf [28]byte // 2x uint16 + 3x uint64 + eb := writeBuf(buf[:]) + eb.uint16(zip64ExtraID) + eb.uint16(24) // size = 3x uint64 + eb.uint64(h.UncompressedSize64) + eb.uint64(h.CompressedSize64) + eb.uint64(h.offset) + h.Extra = append(h.Extra, buf[:]...) + } else { + b.uint32(h.CompressedSize) + b.uint32(h.UncompressedSize) + } + + b.uint16(uint16(len(h.Name))) + b.uint16(uint16(len(h.Extra))) + b.uint16(uint16(len(h.Comment))) + b = b[4:] // skip disk number start and internal file attr (2x uint16) + b.uint32(h.ExternalAttrs) + if h.isZip64() || h.offset > uint32max { + b.uint32(uint32max) + } else { + b.uint32(uint32(h.offset)) + } + if _, err := w.cw.Write(buf[:]); err != nil { + return err + } + if _, err := io.WriteString(w.cw, h.Name); err != nil { + return err + } + if _, err := w.cw.Write(h.Extra); err != nil { + return err + } + if _, err := io.WriteString(w.cw, h.Comment); err != nil { + return err + } + } + end := w.cw.count + + records := uint64(len(w.dir)) + size := uint64(end - start) + offset := uint64(start) + + if f := w.testHookCloseSizeOffset; f != nil { + f(size, offset) + } + + if records >= uint16max || size >= uint32max || offset >= uint32max { + var buf [directory64EndLen + directory64LocLen]byte + b := writeBuf(buf[:]) + + // zip64 end of central directory record + b.uint32(directory64EndSignature) + b.uint64(directory64EndLen - 12) // length minus signature (uint32) and length fields (uint64) + b.uint16(zipVersion45) // version made by + b.uint16(zipVersion45) // version needed to extract + b.uint32(0) // number of this disk + b.uint32(0) // number of the disk with the start of the central directory + b.uint64(records) // total number of entries in the central directory on this disk + b.uint64(records) // total number of entries in the central directory + b.uint64(size) // size of the central directory + b.uint64(offset) // offset of start of central directory with respect to the starting disk number + + // zip64 end of central directory locator + b.uint32(directory64LocSignature) + b.uint32(0) // number of the disk with the start of the zip64 end of central directory + b.uint64(uint64(end)) // relative offset of the zip64 end of central directory record + b.uint32(1) // total number of disks + + if _, err := w.cw.Write(buf[:]); err != nil { + return err + } + + // store max values in the regular end record to signal + // that the zip64 values should be used instead + records = uint16max + size = uint32max + offset = uint32max + } + + // write end record + var buf [directoryEndLen]byte + b := writeBuf(buf[:]) + b.uint32(uint32(directoryEndSignature)) + b = b[4:] // skip over disk number and first disk number (2x uint16) + b.uint16(uint16(records)) // number of entries this disk + b.uint16(uint16(records)) // number of entries total + b.uint32(uint32(size)) // size of directory + b.uint32(uint32(offset)) // start of directory + b.uint16(uint16(len(w.comment))) // byte size of EOCD comment + if _, err := w.cw.Write(buf[:]); err != nil { + return err + } + if _, err := io.WriteString(w.cw, w.comment); err != nil { + return err + } + + return w.cw.w.(*bufio.Writer).Flush() +} + +// Create adds a file to the zip file using the provided name. +// It returns a Writer to which the file contents should be written. +// The file contents will be compressed using the Deflate method. +// The name must be a relative path: it must not start with a drive +// letter (e.g. C:) or leading slash, and only forward slashes are +// allowed. To create a directory instead of a file, add a trailing +// slash to the name. +// The file's contents must be written to the io.Writer before the next +// call to Create, CreateHeader, or Close. +func (w *Writer) Create(name string) (io.Writer, error) { + header := &FileHeader{ + Name: name, + Method: Deflate, + } + return w.CreateHeader(header) +} + +// detectUTF8 reports whether s is a valid UTF-8 string, and whether the string +// must be considered UTF-8 encoding (i.e., not compatible with CP-437, ASCII, +// or any other common encoding). +func detectUTF8(s string) (valid, require bool) { + for i := 0; i < len(s); { + r, size := utf8.DecodeRuneInString(s[i:]) + i += size + // Officially, ZIP uses CP-437, but many readers use the system's + // local character encoding. Most encoding are compatible with a large + // subset of CP-437, which itself is ASCII-like. + // + // Forbid 0x7e and 0x5c since EUC-KR and Shift-JIS replace those + // characters with localized currency and overline characters. + if r < 0x20 || r > 0x7d || r == 0x5c { + if !utf8.ValidRune(r) || (r == utf8.RuneError && size == 1) { + return false, false + } + require = true + } + } + return true, require +} + +// prepare performs the bookkeeping operations required at the start of +// CreateHeader and CreateRaw. +func (w *Writer) prepare(fh *FileHeader) error { + if w.last != nil && !w.last.closed { + if err := w.last.close(); err != nil { + return err + } + } + if len(w.dir) > 0 && w.dir[len(w.dir)-1].FileHeader == fh { + // See https://golang.org/issue/11144 confusion. + return errors.New("archive/zip: invalid duplicate FileHeader") + } + return nil +} + +// CreateHeader adds a file to the zip archive using the provided FileHeader +// for the file metadata. Writer takes ownership of fh and may mutate +// its fields. The caller must not modify fh after calling CreateHeader. +// +// This returns a Writer to which the file contents should be written. +// The file's contents must be written to the io.Writer before the next +// call to Create, CreateHeader, CreateRaw, or Close. +func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) { + if err := w.prepare(fh); err != nil { + return nil, err + } + + // The ZIP format has a sad state of affairs regarding character encoding. + // Officially, the name and comment fields are supposed to be encoded + // in CP-437 (which is mostly compatible with ASCII), unless the UTF-8 + // flag bit is set. However, there are several problems: + // + // * Many ZIP readers still do not support UTF-8. + // * If the UTF-8 flag is cleared, several readers simply interpret the + // name and comment fields as whatever the local system encoding is. + // + // In order to avoid breaking readers without UTF-8 support, + // we avoid setting the UTF-8 flag if the strings are CP-437 compatible. + // However, if the strings require multibyte UTF-8 encoding and is a + // valid UTF-8 string, then we set the UTF-8 bit. + // + // For the case, where the user explicitly wants to specify the encoding + // as UTF-8, they will need to set the flag bit themselves. + utf8Valid1, utf8Require1 := detectUTF8(fh.Name) + utf8Valid2, utf8Require2 := detectUTF8(fh.Comment) + switch { + case fh.NonUTF8: + fh.Flags &^= 0x800 + case (utf8Require1 || utf8Require2) && (utf8Valid1 && utf8Valid2): + fh.Flags |= 0x800 + } + + fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 // preserve compatibility byte + fh.ReaderVersion = zipVersion20 + + // If Modified is set, this takes precedence over MS-DOS timestamp fields. + if !fh.Modified.IsZero() { + // Contrary to the FileHeader.SetModTime method, we intentionally + // do not convert to UTC, because we assume the user intends to encode + // the date using the specified timezone. A user may want this control + // because many legacy ZIP readers interpret the timestamp according + // to the local timezone. + // + // The timezone is only non-UTC if a user directly sets the Modified + // field directly themselves. All other approaches sets UTC. + fh.ModifiedDate, fh.ModifiedTime = timeToMsDosTime(fh.Modified) + + // Use "extended timestamp" format since this is what Info-ZIP uses. + // Nearly every major ZIP implementation uses a different format, + // but at least most seem to be able to understand the other formats. + // + // This format happens to be identical for both local and central header + // if modification time is the only timestamp being encoded. + var mbuf [9]byte // 2*SizeOf(uint16) + SizeOf(uint8) + SizeOf(uint32) + mt := uint32(fh.Modified.Unix()) + eb := writeBuf(mbuf[:]) + eb.uint16(extTimeExtraID) + eb.uint16(5) // Size: SizeOf(uint8) + SizeOf(uint32) + eb.uint8(1) // Flags: ModTime + eb.uint32(mt) // ModTime + fh.Extra = append(fh.Extra, mbuf[:]...) + } + + var ( + ow io.Writer + fw *fileWriter + ) + h := &header{ + FileHeader: fh, + offset: uint64(w.cw.count), + } + + if strings.HasSuffix(fh.Name, "/") { + // Set the compression method to Store to ensure data length is truly zero, + // which the writeHeader method always encodes for the size fields. + // This is necessary as most compression formats have non-zero lengths + // even when compressing an empty string. + fh.Method = Store + fh.Flags &^= 0x8 // we will not write a data descriptor + + // Explicitly clear sizes as they have no meaning for directories. + fh.CompressedSize = 0 + fh.CompressedSize64 = 0 + fh.UncompressedSize = 0 + fh.UncompressedSize64 = 0 + + ow = dirWriter{} + } else { + fh.Flags |= 0x8 // we will write a data descriptor + + fw = &fileWriter{ + zipw: w.cw, + compCount: &countWriter{w: w.cw}, + crc32: crc32.NewIEEE(), + } + comp := w.compressor(fh.Method) + if comp == nil { + return nil, ErrAlgorithm + } + var err error + fw.comp, err = comp(fw.compCount) + if err != nil { + return nil, err + } + fw.rawCount = &countWriter{w: fw.comp} + fw.header = h + ow = fw + } + w.dir = append(w.dir, h) + if err := writeHeader(w.cw, h); err != nil { + return nil, err + } + // If we're creating a directory, fw is nil. + w.last = fw + return ow, nil +} + +func writeHeader(w io.Writer, h *header) error { + const maxUint16 = 1<<16 - 1 + if len(h.Name) > maxUint16 { + return errLongName + } + if len(h.Extra) > maxUint16 { + return errLongExtra + } + + var buf [fileHeaderLen]byte + b := writeBuf(buf[:]) + b.uint32(uint32(fileHeaderSignature)) + b.uint16(h.ReaderVersion) + b.uint16(h.Flags) + b.uint16(h.Method) + b.uint16(h.ModifiedTime) + b.uint16(h.ModifiedDate) + // In raw mode (caller does the compression), the values are either + // written here or in the trailing data descriptor based on the header + // flags. + if h.raw && !h.hasDataDescriptor() { + b.uint32(h.CRC32) + b.uint32(uint32(min64(h.CompressedSize64, uint32max))) + b.uint32(uint32(min64(h.UncompressedSize64, uint32max))) + } else { + // When this package handle the compression, these values are + // always written to the trailing data descriptor. + b.uint32(0) // crc32 + b.uint32(0) // compressed size + b.uint32(0) // uncompressed size + } + b.uint16(uint16(len(h.Name))) + b.uint16(uint16(len(h.Extra))) + if _, err := w.Write(buf[:]); err != nil { + return err + } + if _, err := io.WriteString(w, h.Name); err != nil { + return err + } + _, err := w.Write(h.Extra) + return err +} + +func min64(x, y uint64) uint64 { + if x < y { + return x + } + return y +} + +// Deprecated: CreateHeaderRaw is replaced by CreateRaw (stdlib name). +func (w *Writer) CreateHeaderRaw(fh *FileHeader) (io.Writer, error) { + return w.CreateRaw(fh) +} + +// CreateRaw adds a file to the zip archive using the provided FileHeader and +// returns a Writer to which the file contents should be written. The file's +// contents must be written to the io.Writer before the next call to Create, +// CreateHeader, CreateRaw, or Close. +// +// In contrast to CreateHeader, the bytes passed to Writer are not compressed. +func (w *Writer) CreateRaw(fh *FileHeader) (io.Writer, error) { + if err := w.prepare(fh); err != nil { + return nil, err + } + + fh.CompressedSize = uint32(min64(fh.CompressedSize64, uint32max)) + fh.UncompressedSize = uint32(min64(fh.UncompressedSize64, uint32max)) + + h := &header{ + FileHeader: fh, + offset: uint64(w.cw.count), + raw: true, + } + w.dir = append(w.dir, h) + if err := writeHeader(w.cw, h); err != nil { + return nil, err + } + + if strings.HasSuffix(fh.Name, "/") { + w.last = nil + return dirWriter{}, nil + } + + fw := &fileWriter{ + header: h, + zipw: w.cw, + } + w.last = fw + return fw, nil +} + +// Copy copies the file f (obtained from a Reader) into w. It copies the raw +// form directly bypassing decompression, compression, and validation. +// CHANGE: Optional file name cannot be specified any more due to stdlib api. +func (w *Writer) Copy(f *File) error { + r, err := f.OpenRaw() + if err != nil { + return err + } + fw, err := w.CreateRaw(&f.FileHeader) + if err != nil { + return err + } + _, err = io.Copy(fw, r) + return err +} + +// RegisterCompressor registers or overrides a custom compressor for a specific +// method ID. If a compressor for a given method is not found, Writer will +// default to looking up the compressor at the package level. +func (w *Writer) RegisterCompressor(method uint16, comp Compressor) { + if w.compressors == nil { + w.compressors = make(map[uint16]Compressor) + } + w.compressors[method] = comp +} + +func (w *Writer) compressor(method uint16) Compressor { + comp := w.compressors[method] + if comp == nil { + comp = compressor(method) + } + return comp +} + +type dirWriter struct{} + +func (dirWriter) Write(b []byte) (int, error) { + if len(b) == 0 { + return 0, nil + } + return 0, errors.New("zip: write to directory") +} + +type fileWriter struct { + *header + zipw io.Writer + rawCount *countWriter + comp io.WriteCloser + compCount *countWriter + crc32 hash.Hash32 + closed bool +} + +func (w *fileWriter) Write(p []byte) (int, error) { + if w.closed { + return 0, errors.New("zip: write to closed file") + } + if w.raw { + return w.zipw.Write(p) + } + w.crc32.Write(p) + return w.rawCount.Write(p) +} + +func (w *fileWriter) close() error { + if w.closed { + return errors.New("zip: file closed twice") + } + w.closed = true + if w.raw { + return w.writeDataDescriptor() + } + if err := w.comp.Close(); err != nil { + return err + } + + // update FileHeader + fh := w.header.FileHeader + fh.CRC32 = w.crc32.Sum32() + fh.CompressedSize64 = uint64(w.compCount.count) + fh.UncompressedSize64 = uint64(w.rawCount.count) + + if fh.isZip64() { + fh.CompressedSize = uint32max + fh.UncompressedSize = uint32max + fh.ReaderVersion = zipVersion45 // requires 4.5 - File uses ZIP64 format extensions + } else { + fh.CompressedSize = uint32(fh.CompressedSize64) + fh.UncompressedSize = uint32(fh.UncompressedSize64) + } + + return w.writeDataDescriptor() +} + +func (w *fileWriter) writeDataDescriptor() error { + if !w.hasDataDescriptor() { + return nil + } + // Write data descriptor. This is more complicated than one would + // think, see e.g. comments in zipfile.c:putextended() and + // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7073588. + // The approach here is to write 8 byte sizes if needed without + // adding a zip64 extra in the local header (too late anyway). + var buf []byte + if w.isZip64() { + buf = make([]byte, dataDescriptor64Len) + } else { + buf = make([]byte, dataDescriptorLen) + } + b := writeBuf(buf) + b.uint32(dataDescriptorSignature) // de-facto standard, required by OS X + b.uint32(w.CRC32) + if w.isZip64() { + b.uint64(w.CompressedSize64) + b.uint64(w.UncompressedSize64) + } else { + b.uint32(w.CompressedSize) + b.uint32(w.UncompressedSize) + } + _, err := w.zipw.Write(buf) + return err +} + +type countWriter struct { + w io.Writer + count int64 +} + +func (w *countWriter) Write(p []byte) (int, error) { + n, err := w.w.Write(p) + w.count += int64(n) + return n, err +} + +type nopCloser struct { + io.Writer +} + +func (w nopCloser) Close() error { + return nil +} + +type writeBuf []byte + +func (b *writeBuf) uint8(v uint8) { + (*b)[0] = v + *b = (*b)[1:] +} + +func (b *writeBuf) uint16(v uint16) { + binary.LittleEndian.PutUint16(*b, v) + *b = (*b)[2:] +} + +func (b *writeBuf) uint32(v uint32) { + binary.LittleEndian.PutUint32(*b, v) + *b = (*b)[4:] +} + +func (b *writeBuf) uint64(v uint64) { + binary.LittleEndian.PutUint64(*b, v) + *b = (*b)[8:] +} diff --git a/vendor/github.com/klauspost/pgzip/.travis.yml b/vendor/github.com/klauspost/pgzip/.travis.yml index 6e9fca0b..acfec4bb 100644 --- a/vendor/github.com/klauspost/pgzip/.travis.yml +++ b/vendor/github.com/klauspost/pgzip/.travis.yml @@ -1,19 +1,22 @@ language: go -sudo: false - os: - linux - osx go: - - 1.9.x - - 1.10.x + - 1.13.x + - 1.14.x + - 1.15.x - master -script: - - go test -v -cpu=1,2,4 . - - go test -v -cpu=2 -race -short . +env: + - GO111MODULE=off + +script: + - diff <(gofmt -d .) <(printf "") + - go test -v -cpu=1,2,4 . + - go test -v -cpu=2 -race -short . matrix: allow_failures: diff --git a/vendor/github.com/klauspost/pgzip/LICENSE b/vendor/github.com/klauspost/pgzip/LICENSE index 2bdc0d75..3909da41 100644 --- a/vendor/github.com/klauspost/pgzip/LICENSE +++ b/vendor/github.com/klauspost/pgzip/LICENSE @@ -1,4 +1,4 @@ -The MIT License (MIT) +MIT License Copyright (c) 2014 Klaus Post @@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/vendor/github.com/klauspost/pgzip/README.md b/vendor/github.com/klauspost/pgzip/README.md index 81000996..171b978f 100644 --- a/vendor/github.com/klauspost/pgzip/README.md +++ b/vendor/github.com/klauspost/pgzip/README.md @@ -39,7 +39,6 @@ You might need to get/update the dependencies: ``` go get -u github.com/klauspost/compress -go get -u github.com/klauspost/crc32 ``` Usage @@ -65,7 +64,7 @@ Changes in [github.com/klauspost/compress](https://github.com/klauspost/compress ## Compression The simplest way to use this is to simply do the same as you would when using [compress/gzip](http://golang.org/pkg/compress/gzip). -To change the block size, use the added (*pgzip.Writer).SetConcurrency(blockSize, blocks int) function. With this you can control the approximate size of your blocks, as well as how many you want to be processing in parallel. Default values for this is SetConcurrency(250000, 16), meaning blocks are split at 250000 bytes and up to 16 blocks can be processing at once before the writer blocks. +To change the block size, use the added (*pgzip.Writer).SetConcurrency(blockSize, blocks int) function. With this you can control the approximate size of your blocks, as well as how many you want to be processing in parallel. Default values for this is SetConcurrency(1MB, runtime.GOMAXPROCS(0)), meaning blocks are split at 1 MB and up to the number of CPU threads blocks can be processing at once before the writer blocks. Example: @@ -99,19 +98,19 @@ See my blog post in [Benchmarks of Golang Gzip](https://blog.klauspost.com/go-gz Compression cost is usually about 0.2% with default settings with a block size of 250k. -Example with GOMAXPROC set to 8 (quad core with 8 hyperthreads) +Example with GOMAXPROC set to 32 (16 core CPU) Content is [Matt Mahoneys 10GB corpus](http://mattmahoney.net/dc/10gb.html). Compression level 6. Compressor | MB/sec | speedup | size | size overhead (lower=better) ------------|----------|---------|------|--------- -[gzip](http://golang.org/pkg/compress/gzip) (golang) | 7.21MB/s | 1.0x | 4786608902 | 0% -[gzip](http://github.com/klauspost/compress/gzip) (klauspost) | 10.98MB/s | 1.52x | 4781331645 | -0.11% -[pgzip](https://github.com/klauspost/pgzip) (klauspost) | 50.76MB/s|7.04x | 4784121440 | -0.052% -[bgzf](https://godoc.org/github.com/biogo/hts/bgzf) (biogo) | 38.65MB/s | 5.36x | 4924899484 | 2.889% -[pargzip](https://godoc.org/github.com/golang/build/pargzip) (builder) | 32.00MB/s | 4.44x | 4791226567 | 0.096% +[gzip](http://golang.org/pkg/compress/gzip) (golang) | 15.44MB/s (1 thread) | 1.0x | 4781329307 | 0% +[gzip](http://github.com/klauspost/compress/gzip) (klauspost) | 135.04MB/s (1 thread) | 8.74x | 4894858258 | +2.37% +[pgzip](https://github.com/klauspost/pgzip) (klauspost) | 1573.23MB/s| 101.9x | 4902285651 | +2.53% +[bgzf](https://godoc.org/github.com/biogo/hts/bgzf) (biogo) | 361.40MB/s | 23.4x | 4869686090 | +1.85% +[pargzip](https://godoc.org/github.com/golang/build/pargzip) (builder) | 306.01MB/s | 19.8x | 4786890417 | +0.12% -pgzip also contains a [linear time compression](https://github.com/klauspost/compress#linear-time-compression) mode, that will allow compression at ~150MB per core per second, independent of the content. +pgzip also contains a [linear time compression](https://github.com/klauspost/compress#linear-time-compression-huffman-only) mode, that will allow compression at ~250MB per core per second, independent of the content. See the [complete sheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing) for different content types and compression settings. diff --git a/vendor/github.com/klauspost/pgzip/gunzip.go b/vendor/github.com/klauspost/pgzip/gunzip.go index 93efec71..d1ae730b 100644 --- a/vendor/github.com/klauspost/pgzip/gunzip.go +++ b/vendor/github.com/klauspost/pgzip/gunzip.go @@ -331,6 +331,16 @@ func (z *Reader) killReadAhead() error { // Wait for decompressor to be closed and return error, if any. e, ok := <-z.closeErr z.activeRA = false + + for blk := range z.readAhead { + if blk.b != nil { + z.blockPool <- blk.b + } + } + if cap(z.current) > 0 { + z.blockPool <- z.current + z.current = nil + } if !ok { // Channel is closed, so if there was any error it has already been returned. return nil @@ -418,6 +428,7 @@ func (z *Reader) doReadAhead() { case z.readAhead <- read{b: buf, err: err}: case <-closeReader: // Sent on close, we don't care about the next results + z.blockPool <- buf return } if err != nil { diff --git a/vendor/github.com/klauspost/pgzip/gzip.go b/vendor/github.com/klauspost/pgzip/gzip.go index 85d14e9c..257c4d29 100644 --- a/vendor/github.com/klauspost/pgzip/gzip.go +++ b/vendor/github.com/klauspost/pgzip/gzip.go @@ -11,6 +11,7 @@ import ( "hash" "hash/crc32" "io" + "runtime" "sync" "time" @@ -18,9 +19,9 @@ import ( ) const ( - defaultBlockSize = 256 << 10 + defaultBlockSize = 1 << 20 tailSize = 16384 - defaultBlocks = 16 + defaultBlocks = 4 ) // These constants are copied from the flate package, so that code that imports @@ -68,8 +69,8 @@ type result struct { // With this you can control the approximate size of your blocks, // as well as how many you want to be processing in parallel. // -// Default values for this is SetConcurrency(250000, 16), -// meaning blocks are split at 250000 bytes and up to 16 blocks +// Default values for this is SetConcurrency(defaultBlockSize, runtime.GOMAXPROCS(0)), +// meaning blocks are split at 1 MB and up to the number of CPU threads // can be processing at once before the writer blocks. func (z *Writer) SetConcurrency(blockSize, blocks int) error { if blockSize <= tailSize { @@ -84,7 +85,7 @@ func (z *Writer) SetConcurrency(blockSize, blocks int) error { z.blockSize = blockSize z.results = make(chan result, blocks) z.blocks = blocks - z.dstPool = sync.Pool{New: func() interface{} { return make([]byte, 0, blockSize+(blockSize)>>4) }} + z.dstPool.New = func() interface{} { return make([]byte, 0, blockSize+(blockSize)>>4) } return nil } @@ -115,7 +116,7 @@ func NewWriterLevel(w io.Writer, level int) (*Writer, error) { return nil, fmt.Errorf("gzip: invalid compression level: %d", level) } z := new(Writer) - z.SetConcurrency(defaultBlockSize, defaultBlocks) + z.SetConcurrency(defaultBlockSize, runtime.GOMAXPROCS(0)) z.init(w, level) return z, nil } @@ -174,7 +175,7 @@ func (z *Writer) Reset(w io.Writer) { if z.results != nil && !z.closed { close(z.results) } - z.SetConcurrency(defaultBlockSize, defaultBlocks) + z.SetConcurrency(defaultBlockSize, runtime.GOMAXPROCS(0)) z.init(w, z.level) } @@ -239,36 +240,36 @@ func (z *Writer) writeString(s string) (err error) { // compressCurrent will compress the data currently buffered // This should only be called from the main writer/flush/closer func (z *Writer) compressCurrent(flush bool) { + c := z.currentBuffer + if len(c) > z.blockSize { + // This can never happen through the public interface. + panic("len(z.currentBuffer) > z.blockSize (most likely due to concurrent Write race)") + } + r := result{} r.result = make(chan []byte, 1) r.notifyWritten = make(chan struct{}, 0) + // Reserve a result slot select { case z.results <- r: case <-z.pushedErr: return } - // If block given is more than twice the block size, split it. - c := z.currentBuffer - if len(c) > z.blockSize*2 { - c = c[:z.blockSize] - z.wg.Add(1) - go z.compressBlock(c, z.prevTail, r, false) - z.prevTail = c[len(c)-tailSize:] - z.currentBuffer = z.currentBuffer[z.blockSize:] - z.compressCurrent(flush) - // Last one flushes if needed - return - } - z.wg.Add(1) - go z.compressBlock(c, z.prevTail, r, z.closed) + tail := z.prevTail if len(c) > tailSize { - z.prevTail = c[len(c)-tailSize:] + buf := z.dstPool.Get().([]byte) // Put in .compressBlock + // Copy tail from current buffer before handing the buffer over to the + // compressBlock goroutine. + buf = append(buf[:0], c[len(c)-tailSize:]...) + z.prevTail = buf } else { z.prevTail = nil } - z.currentBuffer = z.dstPool.Get().([]byte) + go z.compressBlock(c, tail, r, z.closed) + + z.currentBuffer = z.dstPool.Get().([]byte) // Put in .compressBlock z.currentBuffer = z.currentBuffer[:0] // Wait if flushing @@ -358,29 +359,37 @@ func (z *Writer) Write(p []byte) (int, error) { // Start receiving data from compressors go func() { listen := z.results + var failed bool for { r, ok := <-listen // If closed, we are finished. if !ok { return } + if failed { + close(r.notifyWritten) + continue + } buf := <-r.result n, err := z.w.Write(buf) if err != nil { z.pushError(err) close(r.notifyWritten) - return + failed = true + continue } if n != len(buf) { z.pushError(fmt.Errorf("gzip: short write %d should be %d", n, len(buf))) + failed = true close(r.notifyWritten) - return + continue } z.dstPool.Put(buf) close(r.notifyWritten) } }() - z.currentBuffer = make([]byte, 0, z.blockSize) + z.currentBuffer = z.dstPool.Get().([]byte) + z.currentBuffer = z.currentBuffer[:0] } q := p for len(q) > 0 { @@ -390,10 +399,13 @@ func (z *Writer) Write(p []byte) (int, error) { } z.digest.Write(q[:length]) z.currentBuffer = append(z.currentBuffer, q[:length]...) - if len(z.currentBuffer) >= z.blockSize { + if len(z.currentBuffer) > z.blockSize { + panic("z.currentBuffer too large (most likely due to concurrent Write race)") + } + if len(z.currentBuffer) == z.blockSize { z.compressCurrent(false) if err := z.checkError(); err != nil { - return len(p) - len(q) - length, err + return len(p) - len(q), err } } z.size += length @@ -410,12 +422,13 @@ func (z *Writer) compressBlock(p, prevTail []byte, r result, closed bool) { close(r.result) z.wg.Done() }() - buf := z.dstPool.Get().([]byte) + buf := z.dstPool.Get().([]byte) // Corresponding Put in .Write's result writer dest := bytes.NewBuffer(buf[:0]) - compressor := z.dictFlatePool.Get().(*flate.Writer) + compressor := z.dictFlatePool.Get().(*flate.Writer) // Put below compressor.ResetDict(dest, prevTail) compressor.Write(p) + z.dstPool.Put(p) // Corresponding Get in .Write and .compressCurrent err := compressor.Flush() if err != nil { @@ -429,7 +442,12 @@ func (z *Writer) compressBlock(p, prevTail []byte, r result, closed bool) { return } } - z.dictFlatePool.Put(compressor) + z.dictFlatePool.Put(compressor) // Get above + + if prevTail != nil { + z.dstPool.Put(prevTail) // Get in .compressCurrent + } + // Read back buffer buf = dest.Bytes() r.result <- buf diff --git a/vendor/github.com/mholt/archiver/.gitignore b/vendor/github.com/mholt/archiver/.gitignore deleted file mode 100644 index ac8f8b25..00000000 --- a/vendor/github.com/mholt/archiver/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.DS_Store -_gitignore -builds/ -*.test -cmd/archiver/archiver diff --git a/vendor/github.com/mholt/archiver/.travis.yml b/vendor/github.com/mholt/archiver/.travis.yml deleted file mode 100644 index 2b39a6e6..00000000 --- a/vendor/github.com/mholt/archiver/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -language: go - -go: - - 1.x - -env: - - CGO_ENABLED=0 - -install: - - go get -t ./... - - go get github.com/golang/lint/golint - - go get github.com/gordonklaus/ineffassign - -script: - - diff <(echo -n) <(gofmt -s -d .) - - ineffassign . - - go vet ./... - - go test ./... - -after_script: - - golint ./... diff --git a/vendor/github.com/mholt/archiver/LICENSE b/vendor/github.com/mholt/archiver/LICENSE deleted file mode 100644 index 315d04f2..00000000 --- a/vendor/github.com/mholt/archiver/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2016 Matthew Holt - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/mholt/archiver/README.md b/vendor/github.com/mholt/archiver/README.md deleted file mode 100644 index f2d94f00..00000000 --- a/vendor/github.com/mholt/archiver/README.md +++ /dev/null @@ -1,110 +0,0 @@ -archiver [![archiver GoDoc](https://img.shields.io/badge/reference-godoc-blue.svg?style=flat-square)](https://godoc.org/github.com/mholt/archiver) [![Linux Build Status](https://img.shields.io/travis/mholt/archiver.svg?style=flat-square&label=linux+build)](https://travis-ci.org/mholt/archiver) [![Windows Build Status](https://img.shields.io/appveyor/ci/mholt/archiver.svg?style=flat-square&label=windows+build)](https://ci.appveyor.com/project/mholt/archiver) -======== - -Package archiver makes it trivially easy to make and extract common archive formats such as .zip, and .tar.gz. Simply name the input and output file(s). - -Files are put into the root of the archive; directories are recursively added, preserving structure. - -The `archiver` command runs the same cross-platform and has no external dependencies (not even libc); powered by the Go standard library, [dsnet/compress](https://github.com/dsnet/compress), [nwaples/rardecode](https://github.com/nwaples/rardecode), and [ulikunitz/xz](https://github.com/ulikunitz/xz). Enjoy! - -Supported formats/extensions: - -- .zip -- .tar -- .tar.gz & .tgz -- .tar.bz2 & .tbz2 -- .tar.xz & .txz -- .tar.lz4 & .tlz4 -- .tar.sz & .tsz -- .rar (open only) - - -## Install - -```bash -go get github.com/mholt/archiver/cmd/archiver -``` - -Or download binaries from the [releases](https://github.com/mholt/archiver/releases) page. - - -## Command Use - -Make a new archive: - -```bash -$ archiver make [archive name] [input files...] -``` - -(At least one input file is required.) - -To extract an archive: - -```bash -$ archiver open [archive name] [destination] -``` - -(The destination path is optional; default is current directory.) - -The archive name must end with a supported file extension—this is how it knows what kind of archive to make. Run `archiver -h` for more help. - - -## Library Use - -```go -import "github.com/mholt/archiver" -``` - -Create a .zip file: - -```go -err := archiver.Zip.Make("output.zip", []string{"file.txt", "folder"}) -``` - -Extract a .zip file: - -```go -err := archiver.Zip.Open("input.zip", "output_folder") -``` - -Working with other file formats is exactly the same, but with [their own Archiver implementations](https://godoc.org/github.com/mholt/archiver#Archiver). - - - -## FAQ - -#### Can I list a file in one folder to go into a different folder in the archive? - -No. This works just like your OS would make an archive in the file explorer: organize your input files to mirror the structure you want in the archive. - - -#### Can it add files to an existing archive? - -Nope. This is a simple tool; it just makes new archives or extracts existing ones. - - -## Project Values - -This project has a few principle-based goals that guide its development: - -- **Do one thing really well.** That is creating and opening archive files. It is not meant to be a replacement for specific archive format tools like tar, zip, etc. that have lots of features and customizability. (Some customizability is OK, but not to the extent that it becomes complicated or error-prone.) - -- **Have good tests.** Changes should be covered by tests. - -- **Limit dependencies.** Keep the package lightweight. - -- **Pure Go.** This means no cgo or other external/system dependencies. This package should be able to stand on its own and cross-compile easily to any platform. - -- **Idiomatic Go.** Keep interfaces small, variable names semantic, vet shows no errors, the linter is generally quiet, etc. - -- **Be elegant.** This package should be elegant to use and its code should be elegant when reading and testing. If it doesn't feel good, fix it up. - -- **Well-documented.** Use comments prudently; explain why non-obvious code is necessary (and use tests to enforce it). Keep the docs updated, and have examples where helpful. - -- **Keep it efficient.** This often means keep it simple. Fast code is valuable. - -- **Consensus.** Contributions should ideally be approved by multiple reviewers before being merged. Generally, avoid merging multi-chunk changes that do not go through at least one or two iterations/reviews. Except for trivial changes, PRs are seldom ready to merge right away. - -- **Have fun contributing.** Coding is awesome! - -We welcome contributions and appreciate your efforts! However, please open issues to discuss any changes before spending the time preparing a pull request. This will save time, reduce frustration, and help coordinate the work. Thank you! diff --git a/vendor/github.com/mholt/archiver/appveyor.yml b/vendor/github.com/mholt/archiver/appveyor.yml deleted file mode 100644 index b970efdd..00000000 --- a/vendor/github.com/mholt/archiver/appveyor.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: "{build}" - -os: Windows Server 2012 R2 - -clone_folder: c:\gopath\src\github.com\mholt\archiver - -environment: - GOPATH: c:\gopath - CGO_ENABLED: 0 - -install: - - rmdir c:\go /s /q - - appveyor DownloadFile https://dl.google.com/go/go1.10.4.windows-amd64.zip - - 7z x go1.10.4.windows-amd64.zip -y -oC:\ > NUL - - go version - - go env - - go get -t ./... - - go get github.com/golang/lint/golint - - go get github.com/gordonklaus/ineffassign - - set PATH=%GOPATH%\bin;%PATH% - -build: off - -test_script: - - go vet ./... - - go test ./... - - ineffassign . - -after_test: - - golint ./... - -deploy: off diff --git a/vendor/github.com/mholt/archiver/archiver.go b/vendor/github.com/mholt/archiver/archiver.go deleted file mode 100644 index 6bf20162..00000000 --- a/vendor/github.com/mholt/archiver/archiver.go +++ /dev/null @@ -1,119 +0,0 @@ -package archiver - -import ( - "fmt" - "io" - "log" - "os" - "path/filepath" - "runtime" - "strings" -) - -// Archiver represent a archive format -type Archiver interface { - // Match checks supported files - Match(filename string) bool - // Make makes an archive file on disk. - Make(destination string, sources []string) error - // Open extracts an archive file on disk. - Open(source, destination string) error - // Write writes an archive to a Writer. - Write(output io.Writer, sources []string) error - // Read reads an archive from a Reader. - Read(input io.Reader, destination string) error -} - -// SupportedFormats contains all supported archive formats -var SupportedFormats = map[string]Archiver{} - -// RegisterFormat adds a supported archive format -func RegisterFormat(name string, format Archiver) { - if _, ok := SupportedFormats[name]; ok { - log.Printf("Format %s already exists, skip!\n", name) - return - } - SupportedFormats[name] = format -} - -// MatchingFormat returns the first archive format that matches -// the given file, or nil if there is no match -func MatchingFormat(fpath string) Archiver { - for _, fmt := range SupportedFormats { - if fmt.Match(fpath) { - return fmt - } - } - return nil -} - -func writeNewFile(fpath string, in io.Reader, fm os.FileMode) error { - err := os.MkdirAll(filepath.Dir(fpath), 0755) - if err != nil { - return fmt.Errorf("%s: making directory for file: %v", fpath, err) - } - - out, err := os.Create(fpath) - if err != nil { - return fmt.Errorf("%s: creating new file: %v", fpath, err) - } - defer out.Close() - - err = out.Chmod(fm) - if err != nil && runtime.GOOS != "windows" { - return fmt.Errorf("%s: changing file mode: %v", fpath, err) - } - - _, err = io.Copy(out, in) - if err != nil { - return fmt.Errorf("%s: writing file: %v", fpath, err) - } - return nil -} - -func writeNewSymbolicLink(fpath string, target string) error { - err := os.MkdirAll(filepath.Dir(fpath), 0755) - if err != nil { - return fmt.Errorf("%s: making directory for file: %v", fpath, err) - } - - err = os.Symlink(target, fpath) - if err != nil { - return fmt.Errorf("%s: making symbolic link for: %v", fpath, err) - } - - return nil -} - -func writeNewHardLink(fpath string, target string) error { - err := os.MkdirAll(filepath.Dir(fpath), 0755) - if err != nil { - return fmt.Errorf("%s: making directory for file: %v", fpath, err) - } - - err = os.Link(target, fpath) - if err != nil { - return fmt.Errorf("%s: making hard link for: %v", fpath, err) - } - - return nil -} - -func mkdir(dirPath string) error { - err := os.MkdirAll(dirPath, 0755) - if err != nil { - return fmt.Errorf("%s: making directory: %v", dirPath, err) - } - return nil -} - -func sanitizeExtractPath(filePath string, destination string) error { - // to avoid zip slip (writing outside of the destination), we resolve - // the target path, and make sure it's nested in the intended - // destination, or bail otherwise. - destpath := filepath.Join(destination, filePath) - if !strings.HasPrefix(destpath, filepath.Clean(destination)) { - return fmt.Errorf("%s: illegal file path", filePath) - } - return nil -} diff --git a/vendor/github.com/mholt/archiver/build.bash b/vendor/github.com/mholt/archiver/build.bash deleted file mode 100644 index b1bef114..00000000 --- a/vendor/github.com/mholt/archiver/build.bash +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -set -ex - -# This script builds archiver for most common platforms. - -export CGO_ENABLED=0 - -cd cmd/archiver -GOOS=linux GOARCH=386 go build -o ../../builds/archiver_linux_386 -GOOS=linux GOARCH=amd64 go build -o ../../builds/archiver_linux_amd64 -GOOS=linux GOARCH=arm go build -o ../../builds/archiver_linux_arm7 -GOOS=linux GOARCH=arm64 go build -o ../../builds/archiver_linux_arm64 -GOOS=darwin GOARCH=amd64 go build -o ../../builds/archiver_mac_amd64 -GOOS=windows GOARCH=386 go build -o ../../builds/archiver_windows_386.exe -GOOS=windows GOARCH=amd64 go build -o ../../builds/archiver_windows_amd64.exe -GOOS=freebsd GOARCH=386 go build -o ../../builds/archiver_freebsd_386 -GOOS=freebsd GOARCH=amd64 go build -o ../../builds/archiver_freebsd_amd64 -GOOS=freebsd GOARCH=arm go build -o ../../builds/archiver_freebsd_arm7 -GOOS=openbsd GOARCH=386 go build -o ../../builds/archiver_openbsd_386 -GOOS=openbsd GOARCH=amd64 go build -o ../../builds/archiver_openbsd_amd64 -cd ../.. \ No newline at end of file diff --git a/vendor/github.com/mholt/archiver/rar.go b/vendor/github.com/mholt/archiver/rar.go deleted file mode 100644 index 3ff61da6..00000000 --- a/vendor/github.com/mholt/archiver/rar.go +++ /dev/null @@ -1,120 +0,0 @@ -package archiver - -import ( - "bytes" - "fmt" - "io" - "os" - "path/filepath" - "strings" - - "github.com/nwaples/rardecode" -) - -// Rar is for RAR archive format -var Rar rarFormat - -func init() { - RegisterFormat("Rar", Rar) -} - -type rarFormat struct{} - -func (rarFormat) Match(filename string) bool { - return strings.HasSuffix(strings.ToLower(filename), ".rar") || isRar(filename) -} - -// isRar checks the file has the RAR 1.5 or 5.0 format signature by reading its -// beginning bytes and matching it -func isRar(rarPath string) bool { - f, err := os.Open(rarPath) - if err != nil { - return false - } - defer f.Close() - - buf := make([]byte, 8) - if n, err := f.Read(buf); err != nil || n < 8 { - return false - } - - return bytes.Equal(buf[:7], []byte("Rar!\x1a\x07\x00")) || // ver 1.5 - bytes.Equal(buf, []byte("Rar!\x1a\x07\x01\x00")) // ver 5.0 -} - -// Write outputs a .rar archive, but this is not implemented because -// RAR is a proprietary format. It is here only for symmetry with -// the other archive formats in this package. -func (rarFormat) Write(output io.Writer, filePaths []string) error { - return fmt.Errorf("write: RAR not implemented (proprietary format)") -} - -// Make makes a .rar archive, but this is not implemented because -// RAR is a proprietary format. It is here only for symmetry with -// the other archive formats in this package. -func (rarFormat) Make(rarPath string, filePaths []string) error { - return fmt.Errorf("make %s: RAR not implemented (proprietary format)", rarPath) -} - -// Read extracts the RAR file read from input and puts the contents -// into destination. -func (rarFormat) Read(input io.Reader, destination string) error { - rr, err := rardecode.NewReader(input, "") - if err != nil { - return fmt.Errorf("read: failed to create reader: %v", err) - } - - return extract(rr, destination) -} - -// Open extracts the RAR file at source and puts the contents -// into destination. -func (rarFormat) Open(source, destination string) error { - rf, err := rardecode.OpenReader(source, "") - if err != nil { - return fmt.Errorf("%s: failed to open file: %v", source, err) - } - defer rf.Close() - - return extract(&rf.Reader, destination) -} - -func extract(rr *rardecode.Reader, destination string) error { - for { - header, err := rr.Next() - if err == io.EOF { - break - } else if err != nil { - return err - } - - err = sanitizeExtractPath(header.Name, destination) - if err != nil { - return err - } - - destpath := filepath.Join(destination, header.Name) - - if header.IsDir { - err = mkdir(destpath) - if err != nil { - return err - } - continue - } - - // if files come before their containing folders, then we must - // create their folders before writing the file - err = mkdir(filepath.Dir(destpath)) - if err != nil { - return err - } - - err = writeNewFile(destpath, rr, header.Mode()) - if err != nil { - return err - } - } - - return nil -} diff --git a/vendor/github.com/mholt/archiver/tar.go b/vendor/github.com/mholt/archiver/tar.go deleted file mode 100644 index ee0c4436..00000000 --- a/vendor/github.com/mholt/archiver/tar.go +++ /dev/null @@ -1,244 +0,0 @@ -package archiver - -import ( - "archive/tar" - "bytes" - "fmt" - "io" - "os" - "path/filepath" - "strconv" - "strings" -) - -// Tar is for Tar format -var Tar tarFormat - -func init() { - RegisterFormat("Tar", Tar) -} - -type tarFormat struct{} - -func (tarFormat) Match(filename string) bool { - return strings.HasSuffix(strings.ToLower(filename), ".tar") || isTar(filename) -} - -const tarBlockSize int = 512 - -// isTar checks the file has the Tar format header by reading its beginning -// block. -func isTar(tarPath string) bool { - f, err := os.Open(tarPath) - if err != nil { - return false - } - defer f.Close() - - buf := make([]byte, tarBlockSize) - if _, err = io.ReadFull(f, buf); err != nil { - return false - } - - return hasTarHeader(buf) -} - -// hasTarHeader checks passed bytes has a valid tar header or not. buf must -// contain at least 512 bytes and if not, it always returns false. -func hasTarHeader(buf []byte) bool { - if len(buf) < tarBlockSize { - return false - } - - b := buf[148:156] - b = bytes.Trim(b, " \x00") // clean up all spaces and null bytes - if len(b) == 0 { - return false // unknown format - } - hdrSum, err := strconv.ParseUint(string(b), 8, 64) - if err != nil { - return false - } - - // According to the go official archive/tar, Sun tar uses signed byte - // values so this calcs both signed and unsigned - var usum uint64 - var sum int64 - for i, c := range buf { - if 148 <= i && i < 156 { - c = ' ' // checksum field itself is counted as branks - } - usum += uint64(uint8(c)) - sum += int64(int8(c)) - } - - if hdrSum != usum && int64(hdrSum) != sum { - return false // invalid checksum - } - - return true -} - -// Write outputs a .tar file to a Writer containing the -// contents of files listed in filePaths. File paths can -// be those of regular files or directories. Regular -// files are stored at the 'root' of the archive, and -// directories are recursively added. -func (tarFormat) Write(output io.Writer, filePaths []string) error { - return writeTar(filePaths, output, "") -} - -// Make creates a .tar file at tarPath containing the -// contents of files listed in filePaths. File paths can -// be those of regular files or directories. Regular -// files are stored at the 'root' of the archive, and -// directories are recursively added. -func (tarFormat) Make(tarPath string, filePaths []string) error { - out, err := os.Create(tarPath) - if err != nil { - return fmt.Errorf("error creating %s: %v", tarPath, err) - } - defer out.Close() - - return writeTar(filePaths, out, tarPath) -} - -func writeTar(filePaths []string, output io.Writer, dest string) error { - tarWriter := tar.NewWriter(output) - defer tarWriter.Close() - - return tarball(filePaths, tarWriter, dest) -} - -// tarball writes all files listed in filePaths into tarWriter, which is -// writing into a file located at dest. -func tarball(filePaths []string, tarWriter *tar.Writer, dest string) error { - for _, fpath := range filePaths { - err := tarFile(tarWriter, fpath, dest) - if err != nil { - return err - } - } - return nil -} - -// tarFile writes the file at source into tarWriter. It does so -// recursively for directories. -func tarFile(tarWriter *tar.Writer, source, dest string) error { - sourceInfo, err := os.Stat(source) - if err != nil { - return fmt.Errorf("%s: stat: %v", source, err) - } - - var baseDir string - if sourceInfo.IsDir() { - baseDir = filepath.Base(source) - } - - return filepath.Walk(source, func(path string, info os.FileInfo, err error) error { - if err != nil { - return fmt.Errorf("error walking to %s: %v", path, err) - } - - header, err := tar.FileInfoHeader(info, path) - if err != nil { - return fmt.Errorf("%s: making header: %v", path, err) - } - - if baseDir != "" { - header.Name = filepath.ToSlash(filepath.Join(baseDir, strings.TrimPrefix(path, source))) - } - - if header.Name == dest { - // our new tar file is inside the directory being archived; skip it - return nil - } - - if info.IsDir() { - header.Name += "/" - } - - err = tarWriter.WriteHeader(header) - if err != nil { - return fmt.Errorf("%s: writing header: %v", path, err) - } - - if info.IsDir() { - return nil - } - - if header.Typeflag == tar.TypeReg { - file, err := os.Open(path) - if err != nil { - return fmt.Errorf("%s: open: %v", path, err) - } - defer file.Close() - - _, err = io.CopyN(tarWriter, file, info.Size()) - if err != nil && err != io.EOF { - return fmt.Errorf("%s: copying contents: %v", path, err) - } - } - return nil - }) -} - -// Read untars a .tar file read from a Reader and puts -// the contents into destination. -func (tarFormat) Read(input io.Reader, destination string) error { - return untar(tar.NewReader(input), destination) -} - -// Open untars source and puts the contents into destination. -func (tarFormat) Open(source, destination string) error { - f, err := os.Open(source) - if err != nil { - return fmt.Errorf("%s: failed to open archive: %v", source, err) - } - defer f.Close() - - return Tar.Read(f, destination) -} - -// untar un-tarballs the contents of tr into destination. -func untar(tr *tar.Reader, destination string) error { - for { - header, err := tr.Next() - if err == io.EOF { - break - } else if err != nil { - return err - } - - if err := untarFile(tr, header, destination); err != nil { - return err - } - } - return nil -} - -// untarFile untars a single file from tr with header header into destination. -func untarFile(tr *tar.Reader, header *tar.Header, destination string) error { - err := sanitizeExtractPath(header.Name, destination) - if err != nil { - return err - } - - destpath := filepath.Join(destination, header.Name) - - switch header.Typeflag { - case tar.TypeDir: - return mkdir(destpath) - case tar.TypeReg, tar.TypeRegA, tar.TypeChar, tar.TypeBlock, tar.TypeFifo: - return writeNewFile(destpath, tr, header.FileInfo().Mode()) - case tar.TypeSymlink: - return writeNewSymbolicLink(destpath, header.Linkname) - case tar.TypeLink: - return writeNewHardLink(destpath, filepath.Join(destination, header.Linkname)) - case tar.TypeXGlobalHeader: - // ignore the pax global header from git generated tarballs - return nil - default: - return fmt.Errorf("%s: unknown type flag: %c", header.Name, header.Typeflag) - } -} diff --git a/vendor/github.com/mholt/archiver/tarbz2.go b/vendor/github.com/mholt/archiver/tarbz2.go deleted file mode 100644 index e0051d3c..00000000 --- a/vendor/github.com/mholt/archiver/tarbz2.go +++ /dev/null @@ -1,106 +0,0 @@ -package archiver - -import ( - "fmt" - "io" - "os" - "strings" - - "github.com/dsnet/compress/bzip2" -) - -// TarBz2 is for TarBz2 format -var TarBz2 tarBz2Format - -func init() { - RegisterFormat("TarBz2", TarBz2) -} - -type tarBz2Format struct{} - -func (tarBz2Format) Match(filename string) bool { - return strings.HasSuffix(strings.ToLower(filename), ".tar.bz2") || - strings.HasSuffix(strings.ToLower(filename), ".tbz2") || - isTarBz2(filename) -} - -// isTarBz2 checks the file has the bzip2 compressed Tar format header by -// reading its beginning block. -func isTarBz2(tarbz2Path string) bool { - f, err := os.Open(tarbz2Path) - if err != nil { - return false - } - defer f.Close() - - bz2r, err := bzip2.NewReader(f, nil) - if err != nil { - return false - } - defer bz2r.Close() - - buf := make([]byte, tarBlockSize) - n, err := bz2r.Read(buf) - if err != nil || n < tarBlockSize { - return false - } - - return hasTarHeader(buf) -} - -// Write outputs a .tar.bz2 file to a Writer containing -// the contents of files listed in filePaths. File paths -// can be those of regular files or directories. Regular -// files are stored at the 'root' of the archive, and -// directories are recursively added. -func (tarBz2Format) Write(output io.Writer, filePaths []string) error { - return writeTarBz2(filePaths, output, "") -} - -// Make creates a .tar.bz2 file at tarbz2Path containing -// the contents of files listed in filePaths. File paths -// can be those of regular files or directories. Regular -// files are stored at the 'root' of the archive, and -// directories are recursively added. -func (tarBz2Format) Make(tarbz2Path string, filePaths []string) error { - out, err := os.Create(tarbz2Path) - if err != nil { - return fmt.Errorf("error creating %s: %v", tarbz2Path, err) - } - defer out.Close() - - return writeTarBz2(filePaths, out, tarbz2Path) -} - -func writeTarBz2(filePaths []string, output io.Writer, dest string) error { - bz2w, err := bzip2.NewWriter(output, nil) - if err != nil { - return fmt.Errorf("error compressing bzip2: %v", err) - } - defer bz2w.Close() - - return writeTar(filePaths, bz2w, dest) -} - -// Read untars a .tar.bz2 file read from a Reader and decompresses -// the contents into destination. -func (tarBz2Format) Read(input io.Reader, destination string) error { - bz2r, err := bzip2.NewReader(input, nil) - if err != nil { - return fmt.Errorf("error decompressing bzip2: %v", err) - } - defer bz2r.Close() - - return Tar.Read(bz2r, destination) -} - -// Open untars source and decompresses the contents into destination. -func (tarBz2Format) Open(source, destination string) error { - f, err := os.Open(source) - if err != nil { - return fmt.Errorf("%s: failed to open archive: %v", source, err) - } - defer f.Close() - - return TarBz2.Read(f, destination) -} diff --git a/vendor/github.com/mholt/archiver/targz.go b/vendor/github.com/mholt/archiver/targz.go deleted file mode 100644 index 6751d49d..00000000 --- a/vendor/github.com/mholt/archiver/targz.go +++ /dev/null @@ -1,98 +0,0 @@ -package archiver - -import ( - "compress/gzip" - "fmt" - "io" - "os" - "strings" -) - -// TarGz is for TarGz format -var TarGz tarGzFormat - -func init() { - RegisterFormat("TarGz", TarGz) -} - -type tarGzFormat struct{} - -func (tarGzFormat) Match(filename string) bool { - return strings.HasSuffix(strings.ToLower(filename), ".tar.gz") || - strings.HasSuffix(strings.ToLower(filename), ".tgz") || - isTarGz(filename) -} - -// isTarGz checks the file has the gzip compressed Tar format header by reading -// its beginning block. -func isTarGz(targzPath string) bool { - f, err := os.Open(targzPath) - if err != nil { - return false - } - defer f.Close() - - gzr, err := gzip.NewReader(f) - if err != nil { - return false - } - defer gzr.Close() - - buf := make([]byte, tarBlockSize) - n, err := gzr.Read(buf) - if err != nil || n < tarBlockSize { - return false - } - - return hasTarHeader(buf) -} - -// Write outputs a .tar.gz file to a Writer containing -// the contents of files listed in filePaths. It works -// the same way Tar does, but with gzip compression. -func (tarGzFormat) Write(output io.Writer, filePaths []string) error { - return writeTarGz(filePaths, output, "") -} - -// Make creates a .tar.gz file at targzPath containing -// the contents of files listed in filePaths. It works -// the same way Tar does, but with gzip compression. -func (tarGzFormat) Make(targzPath string, filePaths []string) error { - out, err := os.Create(targzPath) - if err != nil { - return fmt.Errorf("error creating %s: %v", targzPath, err) - } - defer out.Close() - - return writeTarGz(filePaths, out, targzPath) -} - -func writeTarGz(filePaths []string, output io.Writer, dest string) error { - gzw := gzip.NewWriter(output) - defer gzw.Close() - - return writeTar(filePaths, gzw, dest) -} - -// Read untars a .tar.gz file read from a Reader and decompresses -// the contents into destination. -func (tarGzFormat) Read(input io.Reader, destination string) error { - gzr, err := gzip.NewReader(input) - if err != nil { - return fmt.Errorf("error decompressing: %v", err) - } - defer gzr.Close() - - return Tar.Read(gzr, destination) -} - -// Open untars source and decompresses the contents into destination. -func (tarGzFormat) Open(source, destination string) error { - f, err := os.Open(source) - if err != nil { - return fmt.Errorf("%s: failed to open archive: %v", source, err) - } - defer f.Close() - - return TarGz.Read(f, destination) -} diff --git a/vendor/github.com/mholt/archiver/tarlz4.go b/vendor/github.com/mholt/archiver/tarlz4.go deleted file mode 100644 index 1ddc881f..00000000 --- a/vendor/github.com/mholt/archiver/tarlz4.go +++ /dev/null @@ -1,92 +0,0 @@ -package archiver - -import ( - "fmt" - "io" - "os" - "strings" - - "github.com/pierrec/lz4" -) - -// TarLz4 is for TarLz4 format -var TarLz4 tarLz4Format - -func init() { - RegisterFormat("TarLz4", TarLz4) -} - -type tarLz4Format struct{} - -func (tarLz4Format) Match(filename string) bool { - return strings.HasSuffix(strings.ToLower(filename), ".tar.lz4") || strings.HasSuffix(strings.ToLower(filename), ".tlz4") || isTarLz4(filename) -} - -// isTarLz4 checks the file has the lz4 compressed Tar format header by -// reading its beginning block. -func isTarLz4(tarlz4Path string) bool { - f, err := os.Open(tarlz4Path) - if err != nil { - return false - } - defer f.Close() - - lz4r := lz4.NewReader(f) - buf := make([]byte, tarBlockSize) - n, err := lz4r.Read(buf) - if err != nil || n < tarBlockSize { - return false - } - - return hasTarHeader(buf) -} - -// Write outputs a .tar.lz4 file to a Writer containing -// the contents of files listed in filePaths. File paths -// can be those of regular files or directories. Regular -// files are stored at the 'root' of the archive, and -// directories are recursively added. -func (tarLz4Format) Write(output io.Writer, filePaths []string) error { - return writeTarLz4(filePaths, output, "") -} - -// Make creates a .tar.lz4 file at tarlz4Path containing -// the contents of files listed in filePaths. File paths -// can be those of regular files or directories. Regular -// files are stored at the 'root' of the archive, and -// directories are recursively added. -func (tarLz4Format) Make(tarlz4Path string, filePaths []string) error { - out, err := os.Create(tarlz4Path) - if err != nil { - return fmt.Errorf("error creating %s: %v", tarlz4Path, err) - } - defer out.Close() - - return writeTarLz4(filePaths, out, tarlz4Path) -} - -func writeTarLz4(filePaths []string, output io.Writer, dest string) error { - lz4w := lz4.NewWriter(output) - defer lz4w.Close() - - return writeTar(filePaths, lz4w, dest) -} - -// Read untars a .tar.xz file read from a Reader and decompresses -// the contents into destination. -func (tarLz4Format) Read(input io.Reader, destination string) error { - lz4r := lz4.NewReader(input) - - return Tar.Read(lz4r, destination) -} - -// Open untars source and decompresses the contents into destination. -func (tarLz4Format) Open(source, destination string) error { - f, err := os.Open(source) - if err != nil { - return fmt.Errorf("%s: failed to open archive: %v", source, err) - } - defer f.Close() - - return TarLz4.Read(f, destination) -} diff --git a/vendor/github.com/mholt/archiver/tarsz.go b/vendor/github.com/mholt/archiver/tarsz.go deleted file mode 100644 index 2e290190..00000000 --- a/vendor/github.com/mholt/archiver/tarsz.go +++ /dev/null @@ -1,92 +0,0 @@ -package archiver - -import ( - "fmt" - "io" - "os" - "strings" - - "github.com/golang/snappy" -) - -// TarSz is for TarSz format -var TarSz tarSzFormat - -func init() { - RegisterFormat("TarSz", TarSz) -} - -type tarSzFormat struct{} - -func (tarSzFormat) Match(filename string) bool { - return strings.HasSuffix(strings.ToLower(filename), ".tar.sz") || strings.HasSuffix(strings.ToLower(filename), ".tsz") || isTarSz(filename) -} - -// isTarSz checks the file has the sz compressed Tar format header by -// reading its beginning block. -func isTarSz(tarszPath string) bool { - f, err := os.Open(tarszPath) - if err != nil { - return false - } - defer f.Close() - - szr := snappy.NewReader(f) - buf := make([]byte, tarBlockSize) - n, err := szr.Read(buf) - if err != nil || n < tarBlockSize { - return false - } - - return hasTarHeader(buf) -} - -// Write outputs a .tar.sz file to a Writer containing -// the contents of files listed in filePaths. File paths -// can be those of regular files or directories. Regular -// files are stored at the 'root' of the archive, and -// directories are recursively added. -func (tarSzFormat) Write(output io.Writer, filePaths []string) error { - return writeTarSz(filePaths, output, "") -} - -// Make creates a .tar.sz file at tarszPath containing -// the contents of files listed in filePaths. File paths -// can be those of regular files or directories. Regular -// files are stored at the 'root' of the archive, and -// directories are recursively added. -func (tarSzFormat) Make(tarszPath string, filePaths []string) error { - out, err := os.Create(tarszPath) - if err != nil { - return fmt.Errorf("error creating %s: %v", tarszPath, err) - } - defer out.Close() - - return writeTarSz(filePaths, out, tarszPath) -} - -func writeTarSz(filePaths []string, output io.Writer, dest string) error { - szw := snappy.NewBufferedWriter(output) - defer szw.Close() - - return writeTar(filePaths, szw, dest) -} - -// Read untars a .tar.sz file read from a Reader and decompresses -// the contents into destination. -func (tarSzFormat) Read(input io.Reader, destination string) error { - szr := snappy.NewReader(input) - - return Tar.Read(szr, destination) -} - -// Open untars source and decompresses the contents into destination. -func (tarSzFormat) Open(source, destination string) error { - f, err := os.Open(source) - if err != nil { - return fmt.Errorf("%s: failed to open archive: %v", source, err) - } - defer f.Close() - - return TarSz.Read(f, destination) -} diff --git a/vendor/github.com/mholt/archiver/tarxz.go b/vendor/github.com/mholt/archiver/tarxz.go deleted file mode 100644 index e222fb4a..00000000 --- a/vendor/github.com/mholt/archiver/tarxz.go +++ /dev/null @@ -1,105 +0,0 @@ -package archiver - -import ( - "fmt" - "io" - "os" - "strings" - - "github.com/ulikunitz/xz" -) - -// TarXZ is for TarXZ format -var TarXZ xzFormat - -func init() { - RegisterFormat("TarXZ", TarXZ) -} - -type xzFormat struct{} - -// Match returns whether filename matches this format. -func (xzFormat) Match(filename string) bool { - return strings.HasSuffix(strings.ToLower(filename), ".tar.xz") || - strings.HasSuffix(strings.ToLower(filename), ".txz") || - isTarXz(filename) -} - -// isTarXz checks the file has the xz compressed Tar format header by reading -// its beginning block. -func isTarXz(tarxzPath string) bool { - f, err := os.Open(tarxzPath) - if err != nil { - return false - } - defer f.Close() - - xzr, err := xz.NewReader(f) - if err != nil { - return false - } - - buf := make([]byte, tarBlockSize) - n, err := xzr.Read(buf) - if err != nil || n < tarBlockSize { - return false - } - - return hasTarHeader(buf) -} - -// Write outputs a .tar.xz file to a Writer containing -// the contents of files listed in filePaths. File paths -// can be those of regular files or directories. Regular -// files are stored at the 'root' of the archive, and -// directories are recursively added. -func (xzFormat) Write(output io.Writer, filePaths []string) error { - return writeTarXZ(filePaths, output, "") -} - -// Make creates a .tar.xz file at xzPath containing -// the contents of files listed in filePaths. File -// paths can be those of regular files or directories. -// Regular files are stored at the 'root' of the -// archive, and directories are recursively added. -func (xzFormat) Make(xzPath string, filePaths []string) error { - out, err := os.Create(xzPath) - if err != nil { - return fmt.Errorf("error creating %s: %v", xzPath, err) - } - defer out.Close() - - return writeTarXZ(filePaths, out, xzPath) -} - -func writeTarXZ(filePaths []string, output io.Writer, dest string) error { - xzw, err := xz.NewWriter(output) - if err != nil { - return fmt.Errorf("error compressing xz: %v", err) - } - defer xzw.Close() - - return writeTar(filePaths, xzw, dest) -} - -// Read untars a .tar.xz file read from a Reader and decompresses -// the contents into destination. -func (xzFormat) Read(input io.Reader, destination string) error { - xzr, err := xz.NewReader(input) - if err != nil { - return fmt.Errorf("error decompressing xz: %v", err) - } - - return Tar.Read(xzr, destination) -} - -// Open untars source and decompresses the contents into destination. -func (xzFormat) Open(source, destination string) error { - f, err := os.Open(source) - if err != nil { - return fmt.Errorf("%s: failed to open archive: %v", source, err) - } - defer f.Close() - - return TarXZ.Read(f, destination) -} diff --git a/vendor/github.com/mholt/archiver/v3/.gitignore b/vendor/github.com/mholt/archiver/v3/.gitignore index ac8f8b25..4a87fc1a 100644 --- a/vendor/github.com/mholt/archiver/v3/.gitignore +++ b/vendor/github.com/mholt/archiver/v3/.gitignore @@ -1,5 +1,10 @@ +/arc +/cmd/arc/arc +/dist/ +/vendor/ + .DS_Store _gitignore builds/ *.test -cmd/archiver/archiver +.*.sw* diff --git a/vendor/github.com/mholt/archiver/v3/.goreleaser.yml b/vendor/github.com/mholt/archiver/v3/.goreleaser.yml new file mode 100644 index 00000000..13cc2a67 --- /dev/null +++ b/vendor/github.com/mholt/archiver/v3/.goreleaser.yml @@ -0,0 +1,41 @@ +# This is an example goreleaser.yaml file with some sane defaults. +# Make sure to check the documentation at http://goreleaser.com +project_name: arc +before: + hooks: + # You may remove this if you don't use go modules. + - go mod download + # you may remove this if you don't need go generate + - go generate ./... +builds: + - + env: + - CGO_ENABLED=0 + main: ./cmd/arc + goos: + - linux + - windows + - darwin + goarch: + - 386 + - amd64 + - arm + - arm64 + goarm: + - 6 + - 7 +archives: + - + format: binary + replacements: + darwin: mac +checksum: + name_template: 'checksums.txt' +snapshot: + name_template: "{{ .Tag }}-next" +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' diff --git a/vendor/github.com/mholt/archiver/v3/.prettierrc b/vendor/github.com/mholt/archiver/v3/.prettierrc new file mode 100644 index 00000000..f9f5139c --- /dev/null +++ b/vendor/github.com/mholt/archiver/v3/.prettierrc @@ -0,0 +1,4 @@ +{ + "bracketSpacing": true, + "printWidth": 120, +} diff --git a/vendor/github.com/mholt/archiver/v3/README.md b/vendor/github.com/mholt/archiver/v3/README.md index cbf17d53..c8de5e7e 100644 --- a/vendor/github.com/mholt/archiver/v3/README.md +++ b/vendor/github.com/mholt/archiver/v3/README.md @@ -1,5 +1,4 @@ -archiver [![archiver GoDoc](https://img.shields.io/badge/reference-godoc-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/mholt/archiver?tab=doc) -======== +# archiver [![archiver GoDoc](https://img.shields.io/badge/reference-godoc-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/mholt/archiver?tab=doc) [![Ubuntu-latest](https://github.com/mholt/archiver/actions/workflows/ubuntu-latest.yml/badge.svg)](https://github.com/mholt/archiver/actions/workflows/ubuntu-latest.yml) [![Macos-latest](https://github.com/mholt/archiver/actions/workflows/macos-latest.yml/badge.svg)](https://github.com/mholt/archiver/actions/workflows/macos-latest.yml) [![Windows-latest](https://github.com/mholt/archiver/actions/workflows/windows-latest.yml/badge.svg)](https://github.com/mholt/archiver/actions/workflows/windows-latest.yml) Introducing **Archiver 3.1** - a cross-platform, multi-format archive utility and Go library. A powerful and flexible library meets an elegant CLI in this generic replacement for several platform-specific or format-specific archive utilities. @@ -49,23 +48,76 @@ Files are put into the root of the archive; directories are recursively added, p Tar files can optionally be compressed using any of the above compression formats. +## GoDoc + +See ## Install -To install the runnable binary to your $GOPATH/bin: +### With webi + +[`webi`](https://webinstall.dev/arc) will install `webi` and `arc` to `~/.local/bin/` and update your `PATH`. + +#### Mac, Linux, Raspberry Pi ```bash -$ go install github.com/mholt/archiver/cmd/arc +curl -fsS https://webinstall.dev/arc | bash ``` -Or download binaries from the [releases](https://github.com/mholt/archiver/releases) page. +#### Windows 10 -To use as a dependency in your project: +```pwsh +curl.exe -fsS -A MS https://webinstall.dev/arc | powershell +``` +### With Go + +To install the runnable binary to your \$GOPATH/bin: + +```bash +go install github.com/mholt/archiver/v3/cmd/arc@latest ``` -$ go get github.com/mholt/archiver/v3 + +### Manually + +To install manually + +1. Download the binary for your platform from the [Github Releases](https://github.com/mholt/archiver/releases) page. +2. Move the binary to a location in your path, for example: + - without `sudo`: + ```bash + chmod a+x ~/Downloads/arc_* + mkdir -p ~/.local/bin + mv ~/Downloads/arc_* ~/.local/bin/arc + ``` + - as `root`: + ```bash + chmod a+x ~/Downloads/arc_* + sudo mkdir -p /usr/local/bin + sudo mv ~/Downloads/arc_* /usr/local/bin/arc + ``` +3. If needed, update `~/.bashrc` or `~/.profile` to include add `arc` in your `PATH`, for example: + ``` + echo 'PATH="$HOME:/.local/bin:$PATH"' >> ~/.bashrc + ``` + +## Build from Source + +You can successfully build `arc` with just the go tooling, or with `goreleaser`. + +### With `go` + +```bash +go build cmd/arc/*.go ``` +### Multi-platform with `goreleaser` + +Builds with `goreleaser` will also include version info. + +```bash +goreleaser --snapshot --skip-publish --rm-dist +``` ## Command Use @@ -74,31 +126,32 @@ $ go get github.com/mholt/archiver/v3 ```bash # Syntax: arc archive [archive name] [input files...] -$ arc archive test.tar.gz file1.txt images/file2.jpg folder/subfolder +arc archive test.tar.gz file1.txt images/file2.jpg folder/subfolder ``` (At least one input file is required.) - ### Extract entire archive ```bash # Syntax: arc unarchive [archive name] [destination] -$ arc unarchive test.tar.gz +arc unarchive test.tar.gz ``` (The destination path is optional; default is current directory.) The archive name must end with a supported file extension—this is how it knows what kind of archive to make. Run `arc help` for more help. - ### List archive contents ```bash # Syntax: arc ls [archive name] -$ arc ls caddy_dist.tar.gz +arc ls caddy_dist.tar.gz +``` + +```txt drwxr-xr-x matt staff 0 2018-09-19 15:47:18 -0600 MDT dist/ -rw-r--r-- matt staff 6148 2017-08-07 18:34:22 -0600 MDT dist/.DS_Store -rw-r--r-- matt staff 22481 2018-09-19 15:47:18 -0600 MDT dist/CHANGES.txt @@ -109,23 +162,21 @@ drwxr-xr-x matt staff 0 2018-09-19 15:47:18 -0600 MDT dist/ ... ``` - ### Extract a specific file or folder from an archive ```bash # Syntax: arc extract [archive name] [path in archive] [destination on disk] -$ arc extract test.tar.gz foo/hello.txt extracted/hello.txt +arc extract test.tar.gz foo/hello.txt extracted/hello.txt ``` - ### Compress a single file ```bash # Syntax: arc compress [input file] [output file] -$ arc compress test.txt compressed_test.txt.gz -$ arc compress test.txt gz +arc compress test.txt compressed_test.txt.gz +arc compress test.txt gz ``` For convenience, the output file (second argument) may simply be a compression format (without leading dot), in which case the output filename will be the same as the input filename but with the format extension appended, and the input file will be deleted if successful. @@ -135,23 +186,26 @@ For convenience, the output file (second argument) may simply be a compression f ```bash # Syntax: arc decompress [input file] [output file] -$ arc decompress test.txt.gz original_test.txt -$ arc decompress test.txt.gz +arc decompress test.txt.gz original_test.txt +arc decompress test.txt.gz ``` For convenience, the output file (second argument) may be omitted. In that case, the output filename will have the same name as the input filename, but with the compression extension stripped from the end; and the input file will be deleted if successful. - ### Flags Flags are specified before the subcommand. Use `arc help` or `arc -h` to get usage help and a description of flags with their default values. - - ## Library Use The archiver package allows you to easily create and open archives, walk their contents, extract specific files, compress and decompress files, and even stream archives in and out using pure io.Reader and io.Writer interfaces, without ever needing to touch the disk. +To use as a dependency in your project: + +```bash +go get github.com/mholt/archiver/v3 +``` + ```go import "github.com/mholt/archiver/v3" ``` @@ -209,7 +263,7 @@ for _, fname := range filenames { if err != nil { return err } - + // get file's name for the inside of the archive internalName, err := archiver.NameInArchive(info, fname, fname) if err != nil { @@ -243,7 +297,6 @@ There's a lot more that can be done, too. [See the GoDoc](https://pkg.go.dev/git **Security note: This package does NOT attempt to mitigate zip-slip attacks.** It is [extremely difficult](https://github.com/rubyzip/rubyzip/pull/376) [to do properly](https://github.com/mholt/archiver/pull/65#issuecomment-395988244) and [seemingly impossible to mitigate effectively across platforms](https://github.com/golang/go/issues/20126). [Attempted fixes have broken processing of legitimate files in production](https://github.com/mholt/archiver/pull/70#issuecomment-423267320), rendering the program unusable. Our recommendation instead is to inspect the contents of an untrusted archive before extracting it (this package provides `Walkers`) and decide if you want to proceed with extraction. - ## Project Values This project has a few principle-based goals that guide its development: diff --git a/vendor/github.com/mholt/archiver/v3/SECURITY.md b/vendor/github.com/mholt/archiver/v3/SECURITY.md new file mode 100644 index 00000000..f9157124 --- /dev/null +++ b/vendor/github.com/mholt/archiver/v3/SECURITY.md @@ -0,0 +1,15 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| >= 3.x | :white_check_mark: | +| < 3.0 | :x: | + +## Reporting a Vulnerability + +Please send the details to both of us: + +- AJ ONeal +- Matthew Holt diff --git a/vendor/github.com/mholt/archiver/v3/archiver.go b/vendor/github.com/mholt/archiver/v3/archiver.go index ecf315e0..6fdadadc 100644 --- a/vendor/github.com/mholt/archiver/v3/archiver.go +++ b/vendor/github.com/mholt/archiver/v3/archiver.go @@ -72,6 +72,11 @@ type ExtensionChecker interface { CheckExt(name string) error } +// FilenameChecker validates filenames to prevent path traversal attacks +type FilenameChecker interface { + CheckPath(to, filename string) error +} + // Unarchiver is a type that can extract archive files // into a folder. type Unarchiver interface { @@ -120,6 +125,9 @@ type File struct { type FileInfo struct { os.FileInfo CustomName string + // Stores path to the source. + // Used when reading a symlink. + SourcePath string } // Name returns fi.CustomName if not empty; diff --git a/vendor/github.com/mholt/archiver/v3/azure-pipelines.yml b/vendor/github.com/mholt/archiver/v3/azure-pipelines.yml deleted file mode 100644 index a86a6205..00000000 --- a/vendor/github.com/mholt/archiver/v3/azure-pipelines.yml +++ /dev/null @@ -1,86 +0,0 @@ -trigger: -- master - -strategy: - matrix: - linux: - imageName: ubuntu-16.04 - gorootDir: /usr/local - mac: - imageName: macos-10.13 - gorootDir: /usr/local - windows: - imageName: windows-2019 - gorootDir: C:\ - -pool: - vmImage: $(imageName) - -variables: - GOROOT: $(gorootDir)/go - GOPATH: $(system.defaultWorkingDirectory)/gopath - GOBIN: $(GOPATH)/bin - modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)' - # TODO: Enable modules after upstream dependency zstd supports them - # TODO: modules should be the default in Go 1.13, so this won't be needed - #GO111MODULE: on - -steps: -- bash: | - latestGo=$(curl "https://golang.org/VERSION?m=text") - echo "##vso[task.setvariable variable=LATEST_GO]$latestGo" - echo "Latest Go version: $latestGo" - displayName: "Get latest Go version" - -- bash: | - sudo rm -f $(which go) - echo '##vso[task.prependpath]$(GOBIN)' - echo '##vso[task.prependpath]$(GOROOT)/bin' - mkdir -p '$(modulePath)' - shopt -s extglob - shopt -s dotglob - mv !(gopath) '$(modulePath)' - displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH - -# Install Go (this varies by platform) - -- bash: | - wget "https://dl.google.com/go/$(LATEST_GO).linux-amd64.tar.gz" - sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).linux-amd64.tar.gz" - condition: eq( variables['Agent.OS'], 'Linux' ) - displayName: Install Go on Linux - -- bash: | - wget "https://dl.google.com/go/$(LATEST_GO).darwin-amd64.tar.gz" - sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).darwin-amd64.tar.gz" - condition: eq( variables['Agent.OS'], 'Darwin' ) - displayName: Install Go on macOS - -- powershell: | - Write-Host "Downloading Go... (please be patient, I am very slow)" - (New-Object System.Net.WebClient).DownloadFile("https://dl.google.com/go/$(LATEST_GO).windows-amd64.zip", "$(LATEST_GO).windows-amd64.zip") - Write-Host "Extracting Go... (I'm slow too)" - Expand-Archive "$(LATEST_GO).windows-amd64.zip" -DestinationPath "$(gorootDir)" - condition: eq( variables['Agent.OS'], 'Windows_NT' ) - displayName: Install Go on Windows - -# TODO: When this issue is fixed, replace with installer script: -# https://github.com/golangci/golangci-lint/issues/472 -- script: go get -v github.com/golangci/golangci-lint/cmd/golangci-lint - displayName: Install golangci-lint - -- bash: | - printf "Using go at: $(which go)\n" - printf "Go version: $(go version)\n" - printf "\n\nGo environment:\n\n" - go env - printf "\n\nSystem environment:\n\n" - env - displayName: Print Go version and environment - -- script: | - go get -v -t -d ./... - golangci-lint run -E gofmt -E goimports -E misspell - go test -race ./... - workingDirectory: '$(modulePath)' - displayName: Run tests diff --git a/vendor/github.com/mholt/archiver/v3/error.go b/vendor/github.com/mholt/archiver/v3/error.go new file mode 100644 index 00000000..a46235c6 --- /dev/null +++ b/vendor/github.com/mholt/archiver/v3/error.go @@ -0,0 +1,27 @@ +package archiver + +import ( + "fmt" + "strings" +) + +// IllegalPathError is an error returned when an illegal +// path is detected during the archival process. +// +// By default, only the Filename is showed on error, but you might +// also get the absolute value of the invalid path on the AbsolutePath +// field. +type IllegalPathError struct { + AbsolutePath string + Filename string +} + +func (err *IllegalPathError) Error() string { + return fmt.Sprintf("illegal file path: %s", err.Filename) +} + +// IsIllegalPathError returns true if the provided error is of +// the type IllegalPathError. +func IsIllegalPathError(err error) bool { + return err != nil && strings.Contains(err.Error(), "illegal file path: ") +} diff --git a/vendor/github.com/mholt/archiver/v3/gz.go b/vendor/github.com/mholt/archiver/v3/gz.go index 3922e7a5..650718d0 100644 --- a/vendor/github.com/mholt/archiver/v3/gz.go +++ b/vendor/github.com/mholt/archiver/v3/gz.go @@ -1,12 +1,12 @@ package archiver import ( - "compress/gzip" "fmt" "io" "path/filepath" - pgzip "github.com/klauspost/pgzip" + "github.com/klauspost/compress/gzip" + "github.com/klauspost/pgzip" ) // Gz facilitates gzip compression. diff --git a/vendor/github.com/mholt/archiver/v3/lz4.go b/vendor/github.com/mholt/archiver/v3/lz4.go index daff631d..3d6b0a21 100644 --- a/vendor/github.com/mholt/archiver/v3/lz4.go +++ b/vendor/github.com/mholt/archiver/v3/lz4.go @@ -5,7 +5,7 @@ import ( "io" "path/filepath" - "github.com/pierrec/lz4" + "github.com/pierrec/lz4/v4" ) // Lz4 facilitates LZ4 compression. @@ -16,7 +16,14 @@ type Lz4 struct { // Compress reads in, compresses it, and writes it to out. func (lz *Lz4) Compress(in io.Reader, out io.Writer) error { w := lz4.NewWriter(out) - w.Header.CompressionLevel = lz.CompressionLevel + // TODO archiver v4: use proper lz4.Fast + // bitshifting for backwards compatibility with lz4/v3 + options := []lz4.Option{ + lz4.CompressionLevelOption(lz4.CompressionLevel(1 << (8 + lz.CompressionLevel))), + } + if err := w.Apply(options...); err != nil { + return err + } defer w.Close() _, err := io.Copy(w, in) return err diff --git a/vendor/github.com/mholt/archiver/v3/rar.go b/vendor/github.com/mholt/archiver/v3/rar.go index 84eabda8..35fd60b6 100644 --- a/vendor/github.com/mholt/archiver/v3/rar.go +++ b/vendor/github.com/mholt/archiver/v3/rar.go @@ -40,6 +40,10 @@ type Rar struct { // especially on extraction. ImplicitTopLevelFolder bool + // Strip number of leading paths. This feature is available + // only during unpacking of the entire archive. + StripComponents int + // If true, errors encountered during reading // or writing a single file will be logged and // the operation will continue on remaining files. @@ -60,6 +64,17 @@ func (*Rar) CheckExt(filename string) error { return nil } +// CheckPath ensures that the filename has not been crafted to perform path traversal attacks +func (*Rar) CheckPath(to, filename string) error { + to, _ = filepath.Abs(to) //explicit the destination folder to prevent that 'string.HasPrefix' check can be 'bypassed' when no destination folder is supplied in input + dest := filepath.Join(to, filename) + //prevent path traversal attacks + if !strings.HasPrefix(dest, to) { + return &IllegalPathError{AbsolutePath: dest, Filename: filename} + } + return nil +} + // Unarchive unpacks the .rar file at source to destination. // Destination will be treated as a folder name. It supports // multi-volume archives. @@ -94,7 +109,7 @@ func (r *Rar) Unarchive(source, destination string) error { break } if err != nil { - if r.ContinueOnError { + if r.ContinueOnError || IsIllegalPathError(err) { log.Printf("[ERROR] Reading file in rar archive: %v", err) continue } @@ -145,10 +160,29 @@ func (r *Rar) unrarNext(to string) error { if err != nil { return err // don't wrap error; calling loop must break on io.EOF } + defer f.Close() + header, ok := f.Header.(*rardecode.FileHeader) if !ok { return fmt.Errorf("expected header to be *rardecode.FileHeader but was %T", f.Header) } + + errPath := r.CheckPath(to, header.Name) + if errPath != nil { + return fmt.Errorf("checking path traversal attempt: %v", errPath) + } + + if r.StripComponents > 0 { + if strings.Count(header.Name, "/") < r.StripComponents { + return nil // skip path with fewer components + } + + for i := 0; i < r.StripComponents; i++ { + slash := strings.Index(header.Name, "/") + header.Name = header.Name[slash+1:] + } + } + return r.unrarFile(f, filepath.Join(to, header.Name)) } @@ -363,15 +397,17 @@ func (*Rar) Match(file io.ReadSeeker) (bool, error) { if err != nil { return false, err } - defer file.Seek(currentPos, io.SeekStart) + defer func() { + _, _ = file.Seek(currentPos, io.SeekStart) + }() buf := make([]byte, 8) if n, err := file.Read(buf); err != nil || n < 8 { return false, nil } - hasTarHeader := bytes.Equal(buf[:7], []byte("Rar!\x1a\x07\x00")) || // ver 1.5 + hasRarHeader := bytes.Equal(buf[:7], []byte("Rar!\x1a\x07\x00")) || // ver 1.5 bytes.Equal(buf, []byte("Rar!\x1a\x07\x01\x00")) // ver 5.0 - return hasTarHeader, nil + return hasRarHeader, nil } func (r *Rar) String() string { return "rar" } @@ -402,6 +438,7 @@ var ( _ = Extractor(new(Rar)) _ = Matcher(new(Rar)) _ = ExtensionChecker(new(Rar)) + _ = FilenameChecker(new(Rar)) _ = os.FileInfo(rarFileInfo{}) ) diff --git a/vendor/github.com/mholt/archiver/v3/sz.go b/vendor/github.com/mholt/archiver/v3/sz.go index 39c5865e..02009b52 100644 --- a/vendor/github.com/mholt/archiver/v3/sz.go +++ b/vendor/github.com/mholt/archiver/v3/sz.go @@ -13,7 +13,7 @@ type Snappy struct{} // Compress reads in, compresses it, and writes it to out. func (s *Snappy) Compress(in io.Reader, out io.Writer) error { - w := snappy.NewWriter(out) + w := snappy.NewBufferedWriter(out) defer w.Close() _, err := io.Copy(w, in) return err diff --git a/vendor/github.com/mholt/archiver/v3/tar.go b/vendor/github.com/mholt/archiver/v3/tar.go index e9835315..be898665 100644 --- a/vendor/github.com/mholt/archiver/v3/tar.go +++ b/vendor/github.com/mholt/archiver/v3/tar.go @@ -40,6 +40,10 @@ type Tar struct { // especially on extraction. ImplicitTopLevelFolder bool + // Strip number of leading paths. This feature is available + // only during unpacking of the entire archive. + StripComponents int + // If true, errors encountered during reading // or writing a single file will be logged and // the operation will continue on remaining files. @@ -61,6 +65,17 @@ func (*Tar) CheckExt(filename string) error { return nil } +// CheckPath ensures that the filename has not been crafted to perform path traversal attacks +func (*Tar) CheckPath(to, filename string) error { + to, _ = filepath.Abs(to) //explicit the destination folder to prevent that 'string.HasPrefix' check can be 'bypassed' when no destination folder is supplied in input + dest := filepath.Join(to, filename) + //prevent path traversal attacks + if !strings.HasPrefix(dest, to) { + return &IllegalPathError{AbsolutePath: dest, Filename: filename} + } + return nil +} + // Archive creates a tarball file at destination containing // the files listed in sources. The destination must end with // ".tar". File paths can be those of regular files or @@ -150,7 +165,7 @@ func (t *Tar) Unarchive(source, destination string) error { break } if err != nil { - if t.ContinueOnError { + if t.ContinueOnError || IsIllegalPathError(err) { log.Printf("[ERROR] Reading file in tar archive: %v", err) continue } @@ -206,29 +221,44 @@ func (t *Tar) addTopLevelFolder(sourceArchive, destination string) (string, erro return destination, nil } -func (t *Tar) untarNext(to string) error { +func (t *Tar) untarNext(destination string) error { f, err := t.Read() if err != nil { return err // don't wrap error; calling loop must break on io.EOF } + defer f.Close() + header, ok := f.Header.(*tar.Header) if !ok { return fmt.Errorf("expected header to be *tar.Header but was %T", f.Header) } - return t.untarFile(f, filepath.Join(to, header.Name)) + + errPath := t.CheckPath(destination, header.Name) + if errPath != nil { + return fmt.Errorf("checking path traversal attempt: %v", errPath) + } + + if t.StripComponents > 0 { + if strings.Count(header.Name, "/") < t.StripComponents { + return nil // skip path with fewer components + } + + for i := 0; i < t.StripComponents; i++ { + slash := strings.Index(header.Name, "/") + header.Name = header.Name[slash+1:] + } + } + return t.untarFile(f, destination, header) } -func (t *Tar) untarFile(f File, to string) error { +func (t *Tar) untarFile(f File, destination string, hdr *tar.Header) error { + to := filepath.Join(destination, hdr.Name) + // do not overwrite existing files, if configured if !f.IsDir() && !t.OverwriteExisting && fileExists(to) { return fmt.Errorf("file already exists: %s", to) } - hdr, ok := f.Header.(*tar.Header) - if !ok { - return fmt.Errorf("expected header to be *tar.Header but was %T", f.Header) - } - switch hdr.Typeflag { case tar.TypeDir: return mkdir(to, f.Mode()) @@ -237,7 +267,7 @@ func (t *Tar) untarFile(f File, to string) error { case tar.TypeSymlink: return writeNewSymbolicLink(to, hdr.Linkname) case tar.TypeLink: - return writeNewHardLink(to, filepath.Join(to, hdr.Linkname)) + return writeNewHardLink(to, filepath.Join(destination, hdr.Linkname)) case tar.TypeXGlobalHeader: return nil // ignore the pax global header from git-generated tarballs default: @@ -297,6 +327,7 @@ func (t *Tar) writeWalk(source, topLevelFolder, destination string) error { FileInfo: FileInfo{ FileInfo: info, CustomName: nameInArchive, + SourcePath: fpath, }, ReadCloser: file, }) @@ -342,10 +373,14 @@ func (t *Tar) Write(f File) error { var linkTarget string if isSymlink(f) { + fi, ok := f.FileInfo.(FileInfo) + if !ok { + return fmt.Errorf("failed to cast fs.FileInfo to archiver.FileInfo: %v", f) + } var err error - linkTarget, err = os.Readlink(f.Name()) + linkTarget, err = os.Readlink(fi.SourcePath) if err != nil { - return fmt.Errorf("%s: readlink: %v", f.Name(), err) + return fmt.Errorf("%s: readlink: %v", fi.SourcePath, err) } } @@ -356,7 +391,7 @@ func (t *Tar) Write(f File) error { err = t.tw.WriteHeader(hdr) if err != nil { - return fmt.Errorf("%s: writing header: %v", hdr.Name, err) + return fmt.Errorf("%s: writing header: %w", hdr.Name, err) } if f.IsDir() { @@ -369,7 +404,7 @@ func (t *Tar) Write(f File) error { } _, err := io.Copy(t.tw, f) if err != nil { - return fmt.Errorf("%s: copying contents: %v", f.Name(), err) + return fmt.Errorf("%s: copying contents: %w", f.Name(), err) } } @@ -513,9 +548,14 @@ func (t *Tar) Extract(source, target, destination string) error { if err != nil { return fmt.Errorf("relativizing paths: %v", err) } - joined := filepath.Join(destination, end) + th.Name = end + + // relativize any hardlink names + if th.Typeflag == tar.TypeLink { + th.Linkname = filepath.Join(filepath.Base(filepath.Dir(th.Linkname)), filepath.Base(th.Linkname)) + } - err = t.untarFile(f, joined) + err = t.untarFile(f, destination, th) if err != nil { return fmt.Errorf("extracting file %s: %v", th.Name, err) } @@ -544,7 +584,9 @@ func (*Tar) Match(file io.ReadSeeker) (bool, error) { if err != nil { return false, err } - defer file.Seek(currentPos, io.SeekStart) + defer func() { + _, _ = file.Seek(currentPos, io.SeekStart) + }() buf := make([]byte, tarBlockSize) if _, err = io.ReadFull(file, buf); err != nil { @@ -610,6 +652,7 @@ var ( _ = Extractor(new(Tar)) _ = Matcher(new(Tar)) _ = ExtensionChecker(new(Tar)) + _ = FilenameChecker(new(Tar)) ) // DefaultTar is a default instance that is conveniently ready to use. diff --git a/vendor/github.com/mholt/archiver/v3/targz.go b/vendor/github.com/mholt/archiver/v3/targz.go index 07777d0c..283fd01b 100644 --- a/vendor/github.com/mholt/archiver/v3/targz.go +++ b/vendor/github.com/mholt/archiver/v3/targz.go @@ -1,12 +1,12 @@ package archiver import ( - "compress/gzip" "fmt" "io" "strings" - pgzip "github.com/klauspost/pgzip" + "github.com/klauspost/compress/gzip" + "github.com/klauspost/pgzip" ) // TarGz facilitates gzip compression diff --git a/vendor/github.com/mholt/archiver/v3/tarlz4.go b/vendor/github.com/mholt/archiver/v3/tarlz4.go index 4a178f69..42cbc90b 100644 --- a/vendor/github.com/mholt/archiver/v3/tarlz4.go +++ b/vendor/github.com/mholt/archiver/v3/tarlz4.go @@ -5,7 +5,7 @@ import ( "io" "strings" - "github.com/pierrec/lz4" + "github.com/pierrec/lz4/v4" ) // TarLz4 facilitates lz4 compression @@ -84,7 +84,14 @@ func (tlz4 *TarLz4) wrapWriter() { var lz4w *lz4.Writer tlz4.Tar.writerWrapFn = func(w io.Writer) (io.Writer, error) { lz4w = lz4.NewWriter(w) - lz4w.Header.CompressionLevel = tlz4.CompressionLevel + // TODO archiver v4: use proper lz4.Fast + // bitshifting for backwards compatibility with lz4/v3 + options := []lz4.Option{ + lz4.CompressionLevelOption(lz4.CompressionLevel(1 << (8 + tlz4.CompressionLevel))), + } + if err := lz4w.Apply(options...); err != nil { + return lz4w, err + } return lz4w, nil } tlz4.Tar.cleanupWrapFn = func() { diff --git a/vendor/github.com/mholt/archiver/v3/tarsz.go b/vendor/github.com/mholt/archiver/v3/tarsz.go index 0569e664..ee3808e6 100644 --- a/vendor/github.com/mholt/archiver/v3/tarsz.go +++ b/vendor/github.com/mholt/archiver/v3/tarsz.go @@ -77,7 +77,7 @@ func (tsz *TarSz) Extract(source, target, destination string) error { func (tsz *TarSz) wrapWriter() { var sw *snappy.Writer tsz.Tar.writerWrapFn = func(w io.Writer) (io.Writer, error) { - sw = snappy.NewWriter(w) + sw = snappy.NewBufferedWriter(w) return sw, nil } tsz.Tar.cleanupWrapFn = func() { diff --git a/vendor/github.com/mholt/archiver/v3/zip.go b/vendor/github.com/mholt/archiver/v3/zip.go index 192bf607..c6af8efb 100644 --- a/vendor/github.com/mholt/archiver/v3/zip.go +++ b/vendor/github.com/mholt/archiver/v3/zip.go @@ -1,16 +1,36 @@ package archiver import ( - "archive/zip" "bytes" "compress/flate" "fmt" "io" + "io/ioutil" "log" "os" "path" "path/filepath" "strings" + + "github.com/dsnet/compress/bzip2" + "github.com/klauspost/compress/zip" + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" +) + +// ZipCompressionMethod Compression type +type ZipCompressionMethod uint16 + +// Compression methods. +// see https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT. +// Note LZMA: Disabled - because 7z isn't able to unpack ZIP+LZMA ZIP+LZMA2 archives made this way - and vice versa. +const ( + Store ZipCompressionMethod = 0 + Deflate ZipCompressionMethod = 8 + BZIP2 ZipCompressionMethod = 12 + LZMA ZipCompressionMethod = 14 + ZSTD ZipCompressionMethod = 93 + XZ ZipCompressionMethod = 95 ) // Zip provides facilities for operating ZIP archives. @@ -50,14 +70,21 @@ type Zip struct { // especially on extraction. ImplicitTopLevelFolder bool + // Strip number of leading paths. This feature is available + // only during unpacking of the entire archive. + StripComponents int + // If true, errors encountered during reading // or writing a single file will be logged and // the operation will continue on remaining files. ContinueOnError bool - zw *zip.Writer - zr *zip.Reader - ridx int + // Compression algorithm + FileMethod ZipCompressionMethod + zw *zip.Writer + zr *zip.Reader + ridx int + //decinitialized bool } // CheckExt ensures the file extension matches the format. @@ -68,6 +95,43 @@ func (*Zip) CheckExt(filename string) error { return nil } +// Registering a global decompressor is not reentrant and may panic +func registerDecompressor(zr *zip.Reader) { + // register zstd decompressor + zr.RegisterDecompressor(uint16(ZSTD), func(r io.Reader) io.ReadCloser { + zr, err := zstd.NewReader(r) + if err != nil { + return nil + } + return zr.IOReadCloser() + }) + zr.RegisterDecompressor(uint16(BZIP2), func(r io.Reader) io.ReadCloser { + bz2r, err := bzip2.NewReader(r, nil) + if err != nil { + return nil + } + return bz2r + }) + zr.RegisterDecompressor(uint16(XZ), func(r io.Reader) io.ReadCloser { + xr, err := xz.NewReader(r) + if err != nil { + return nil + } + return ioutil.NopCloser(xr) + }) +} + +// CheckPath ensures the file extension matches the format. +func (*Zip) CheckPath(to, filename string) error { + to, _ = filepath.Abs(to) //explicit the destination folder to prevent that 'string.HasPrefix' check can be 'bypassed' when no destination folder is supplied in input + dest := filepath.Join(to, filename) + //prevent path traversal attacks + if !strings.HasPrefix(dest, to) { + return &IllegalPathError{AbsolutePath: dest, Filename: filename} + } + return nil +} + // Archive creates a .zip file at destination containing // the files listed in sources. The destination must end // with ".zip". File paths can be those of regular files @@ -165,7 +229,7 @@ func (z *Zip) Unarchive(source, destination string) error { break } if err != nil { - if z.ContinueOnError { + if z.ContinueOnError || IsIllegalPathError(err) { log.Printf("[ERROR] Reading file in zip archive: %v", err) continue } @@ -182,15 +246,31 @@ func (z *Zip) extractNext(to string) error { return err // don't wrap error; calling loop must break on io.EOF } defer f.Close() - return z.extractFile(f, to) -} -func (z *Zip) extractFile(f File, to string) error { header, ok := f.Header.(zip.FileHeader) if !ok { return fmt.Errorf("expected header to be zip.FileHeader but was %T", f.Header) } + errPath := z.CheckPath(to, header.Name) + if errPath != nil { + return fmt.Errorf("checking path traversal attempt: %v", errPath) + } + + if z.StripComponents > 0 { + if strings.Count(header.Name, "/") < z.StripComponents { + return nil // skip path with fewer components + } + + for i := 0; i < z.StripComponents; i++ { + slash := strings.Index(header.Name, "/") + header.Name = header.Name[slash+1:] + } + } + return z.extractFile(f, to, &header) +} + +func (z *Zip) extractFile(f File, to string, header *zip.FileHeader) error { to = filepath.Join(to, header.Name) // if a directory, no content; simply make the directory and return @@ -270,6 +350,7 @@ func (z *Zip) writeWalk(source, topLevelFolder, destination string) error { FileInfo: FileInfo{ FileInfo: info, CustomName: nameInArchive, + SourcePath: fpath, }, ReadCloser: file, }) @@ -292,6 +373,20 @@ func (z *Zip) Create(out io.Writer) error { return flate.NewWriter(out, z.CompressionLevel) }) } + switch z.FileMethod { + case BZIP2: + z.zw.RegisterCompressor(uint16(BZIP2), func(out io.Writer) (io.WriteCloser, error) { + return bzip2.NewWriter(out, &bzip2.WriterConfig{Level: z.CompressionLevel}) + }) + case ZSTD: + z.zw.RegisterCompressor(uint16(ZSTD), func(out io.Writer) (io.WriteCloser, error) { + return zstd.NewWriter(out) + }) + case XZ: + z.zw.RegisterCompressor(uint16(XZ), func(out io.Writer) (io.WriteCloser, error) { + return xz.NewWriter(out) + }) + } return nil } @@ -320,13 +415,13 @@ func (z *Zip) Write(f File) error { if _, ok := compressedFormats[ext]; ok && z.SelectiveCompression { header.Method = zip.Store } else { - header.Method = zip.Deflate + header.Method = uint16(z.FileMethod) } } writer, err := z.zw.CreateHeader(header) if err != nil { - return fmt.Errorf("%s: making header: %v", f.Name(), err) + return fmt.Errorf("%s: making header: %w", f.Name(), err) } return z.writeFile(f, writer) @@ -337,14 +432,18 @@ func (z *Zip) writeFile(f File, writer io.Writer) error { return nil // directories have no contents } if isSymlink(f) { + fi, ok := f.FileInfo.(FileInfo) + if !ok { + return fmt.Errorf("failed to cast fs.FileInfo to archiver.FileInfo: %v", f) + } // file body for symlinks is the symlink target - linkTarget, err := os.Readlink(f.Name()) + linkTarget, err := os.Readlink(fi.SourcePath) if err != nil { - return fmt.Errorf("%s: readlink: %v", f.Name(), err) + return fmt.Errorf("%s: readlink: %v", fi.SourcePath, err) } _, err = writer.Write([]byte(filepath.ToSlash(linkTarget))) if err != nil { - return fmt.Errorf("%s: writing symlink target: %v", f.Name(), err) + return fmt.Errorf("%s: writing symlink target: %v", fi.SourcePath, err) } return nil } @@ -354,7 +453,7 @@ func (z *Zip) writeFile(f File, writer io.Writer) error { } _, err := io.Copy(writer, f) if err != nil { - return fmt.Errorf("%s: copying contents: %v", f.Name(), err) + return fmt.Errorf("%s: copying contents: %w", f.Name(), err) } return nil @@ -376,6 +475,7 @@ func (z *Zip) Open(in io.Reader, size int64) error { if err != nil { return fmt.Errorf("creating reader: %v", err) } + registerDecompressor(z.zr) z.ridx = 0 return nil } @@ -432,11 +532,13 @@ func (z *Zip) Walk(archive string, walkFn WalkFunc) error { return fmt.Errorf("opening zip reader: %v", err) } defer zr.Close() - + registerDecompressor(&zr.Reader) for _, zf := range zr.File { zfrc, err := zf.Open() if err != nil { - zfrc.Close() + if zfrc != nil { + zfrc.Close() + } if z.ContinueOnError { log.Printf("[ERROR] Opening %s: %v", zf.Name, err) continue @@ -501,7 +603,7 @@ func (z *Zip) Extract(source, target, destination string) error { } joined := filepath.Join(destination, end) - err = z.extractFile(f, joined) + err = z.extractFile(f, joined, &zfh) if err != nil { return fmt.Errorf("extracting file %s: %v", zfh.Name, err) } @@ -530,7 +632,9 @@ func (*Zip) Match(file io.ReadSeeker) (bool, error) { if err != nil { return false, err } - defer file.Seek(currentPos, io.SeekStart) + defer func() { + _, _ = file.Seek(currentPos, io.SeekStart) + }() buf := make([]byte, 4) if n, err := file.Read(buf); err != nil || n < 4 { @@ -547,6 +651,7 @@ func NewZip() *Zip { CompressionLevel: flate.DefaultCompression, MkdirAll: true, SelectiveCompression: true, + FileMethod: Deflate, } } @@ -560,6 +665,7 @@ var ( _ = Extractor(new(Zip)) _ = Matcher(new(Zip)) _ = ExtensionChecker(new(Zip)) + _ = FilenameChecker(new(Zip)) ) // compressedFormats is a (non-exhaustive) set of lowercased diff --git a/vendor/github.com/mholt/archiver/v3/zstd.go b/vendor/github.com/mholt/archiver/v3/zstd.go index 3955628c..60c11efc 100644 --- a/vendor/github.com/mholt/archiver/v3/zstd.go +++ b/vendor/github.com/mholt/archiver/v3/zstd.go @@ -10,11 +10,13 @@ import ( // Zstd facilitates Zstandard compression. type Zstd struct { + EncoderOptions []zstd.EOption + DecoderOptions []zstd.DOption } // Compress reads in, compresses it, and writes it to out. func (zs *Zstd) Compress(in io.Reader, out io.Writer) error { - w, err := zstd.NewWriter(out) + w, err := zstd.NewWriter(out, zs.EncoderOptions...) if err != nil { return err } @@ -25,7 +27,7 @@ func (zs *Zstd) Compress(in io.Reader, out io.Writer) error { // Decompress reads in, decompresses it, and writes it to out. func (zs *Zstd) Decompress(in io.Reader, out io.Writer) error { - r, err := zstd.NewReader(in) + r, err := zstd.NewReader(in, zs.DecoderOptions...) if err != nil { return err } diff --git a/vendor/github.com/mholt/archiver/zip.go b/vendor/github.com/mholt/archiver/zip.go deleted file mode 100644 index 9d20bc1b..00000000 --- a/vendor/github.com/mholt/archiver/zip.go +++ /dev/null @@ -1,238 +0,0 @@ -// Package archiver makes it super easy to create and open .zip, -// .tar.gz, and .tar.bz2 files. -package archiver - -import ( - "archive/zip" - "bytes" - "fmt" - "io" - "io/ioutil" - "os" - "path" - "path/filepath" - "strings" -) - -// Zip is for Zip format -var Zip zipFormat - -func init() { - RegisterFormat("Zip", Zip) -} - -type zipFormat struct{} - -func (zipFormat) Match(filename string) bool { - return strings.HasSuffix(strings.ToLower(filename), ".zip") || isZip(filename) -} - -// isZip checks the file has the Zip format signature by reading its beginning -// bytes and matching it against "PK\x03\x04" -func isZip(zipPath string) bool { - f, err := os.Open(zipPath) - if err != nil { - return false - } - defer f.Close() - - buf := make([]byte, 4) - if n, err := f.Read(buf); err != nil || n < 4 { - return false - } - - return bytes.Equal(buf, []byte("PK\x03\x04")) -} - -// Write outputs a .zip file to the given writer with -// the contents of files listed in filePaths. File paths -// can be those of regular files or directories. Regular -// files are stored at the 'root' of the archive, and -// directories are recursively added. -// -// Files with an extension for formats that are already -// compressed will be stored only, not compressed. -func (zipFormat) Write(output io.Writer, filePaths []string) error { - w := zip.NewWriter(output) - for _, fpath := range filePaths { - if err := zipFile(w, fpath); err != nil { - w.Close() - return err - } - } - - return w.Close() -} - -// Make creates a .zip file in the location zipPath containing -// the contents of files listed in filePaths. File paths -// can be those of regular files or directories. Regular -// files are stored at the 'root' of the archive, and -// directories are recursively added. -// -// Files with an extension for formats that are already -// compressed will be stored only, not compressed. -func (zipFormat) Make(zipPath string, filePaths []string) error { - out, err := os.Create(zipPath) - if err != nil { - return fmt.Errorf("error creating %s: %v", zipPath, err) - } - defer out.Close() - - return Zip.Write(out, filePaths) -} - -func zipFile(w *zip.Writer, source string) error { - sourceInfo, err := os.Stat(source) - if err != nil { - return fmt.Errorf("%s: stat: %v", source, err) - } - - var baseDir string - if sourceInfo.IsDir() { - baseDir = filepath.Base(source) - } - - return filepath.Walk(source, func(fpath string, info os.FileInfo, err error) error { - if err != nil { - return fmt.Errorf("walking to %s: %v", fpath, err) - } - - header, err := zip.FileInfoHeader(info) - if err != nil { - return fmt.Errorf("%s: getting header: %v", fpath, err) - } - - if baseDir != "" { - name, err := filepath.Rel(source, fpath) - if err != nil { - return err - } - header.Name = path.Join(baseDir, filepath.ToSlash(name)) - } - - if info.IsDir() { - header.Name += "/" - header.Method = zip.Store - } else { - ext := strings.ToLower(path.Ext(header.Name)) - if _, ok := compressedFormats[ext]; ok { - header.Method = zip.Store - } else { - header.Method = zip.Deflate - } - } - - writer, err := w.CreateHeader(header) - if err != nil { - return fmt.Errorf("%s: making header: %v", fpath, err) - } - - if info.IsDir() { - return nil - } - - if header.Mode().IsRegular() { - file, err := os.Open(fpath) - if err != nil { - return fmt.Errorf("%s: opening: %v", fpath, err) - } - defer file.Close() - - _, err = io.CopyN(writer, file, info.Size()) - if err != nil && err != io.EOF { - return fmt.Errorf("%s: copying contents: %v", fpath, err) - } - } - - return nil - }) -} - -// Read unzips the .zip file read from the input Reader into destination. -func (zipFormat) Read(input io.Reader, destination string) error { - buf, err := ioutil.ReadAll(input) - if err != nil { - return err - } - - rdr := bytes.NewReader(buf) - r, err := zip.NewReader(rdr, rdr.Size()) - if err != nil { - return err - } - - return unzipAll(r, destination) -} - -// Open unzips the .zip file at source into destination. -func (zipFormat) Open(source, destination string) error { - r, err := zip.OpenReader(source) - if err != nil { - return err - } - defer r.Close() - - return unzipAll(&r.Reader, destination) -} - -func unzipAll(r *zip.Reader, destination string) error { - for _, zf := range r.File { - if err := unzipFile(zf, destination); err != nil { - return err - } - } - - return nil -} - -func unzipFile(zf *zip.File, destination string) error { - err := sanitizeExtractPath(zf.Name, destination) - if err != nil { - return err - } - - if strings.HasSuffix(zf.Name, "/") { - return mkdir(filepath.Join(destination, zf.Name)) - } - - rc, err := zf.Open() - if err != nil { - return fmt.Errorf("%s: open compressed file: %v", zf.Name, err) - } - defer rc.Close() - - return writeNewFile(filepath.Join(destination, zf.Name), rc, zf.FileInfo().Mode()) -} - -// compressedFormats is a (non-exhaustive) set of lowercased -// file extensions for formats that are typically already -// compressed. Compressing already-compressed files often -// results in a larger file, so when possible, we check this -// set to avoid that. -var compressedFormats = map[string]struct{}{ - ".7z": {}, - ".avi": {}, - ".bz2": {}, - ".cab": {}, - ".gif": {}, - ".gz": {}, - ".jar": {}, - ".jpeg": {}, - ".jpg": {}, - ".lz": {}, - ".lzma": {}, - ".mov": {}, - ".mp3": {}, - ".mp4": {}, - ".mpeg": {}, - ".mpg": {}, - ".png": {}, - ".rar": {}, - ".tbz2": {}, - ".tgz": {}, - ".txz": {}, - ".xz": {}, - ".zip": {}, - ".zipx": {}, -} diff --git a/vendor/github.com/nwaples/rardecode/archive.go b/vendor/github.com/nwaples/rardecode/archive.go index 8929f126..f8787511 100644 --- a/vendor/github.com/nwaples/rardecode/archive.go +++ b/vendor/github.com/nwaples/rardecode/archive.go @@ -137,12 +137,13 @@ func findSig(br *bufio.Reader) (int, error) { // files in a multi-volume archive type volume struct { fileBlockReader - f *os.File // current file handle - br *bufio.Reader // buffered reader for current volume file - dir string // volume directory - file string // current volume file - num int // volume number - old bool // uses old naming scheme + f *os.File // current file handle + br *bufio.Reader // buffered reader for current volume file + dir string // volume directory + file string // current volume file (not including directory) + files []string // full path names for current volume files processed + num int // volume number + old bool // uses old naming scheme } // nextVolName updates name to the next filename in the archive. @@ -258,6 +259,7 @@ func (v *volume) next() (*fileBlockHeader, error) { if v.version() != ver { return nil, errVerMismatch } + v.files = append(v.files, v.dir+v.file) v.reset() // reset encryption } } @@ -284,6 +286,7 @@ func openVolume(name, password string) (*volume, error) { v.f.Close() return nil, err } + v.files = append(v.files, name) return v, nil } diff --git a/vendor/github.com/nwaples/rardecode/huffman.go b/vendor/github.com/nwaples/rardecode/huffman.go index eb289b40..4acb69d5 100644 --- a/vendor/github.com/nwaples/rardecode/huffman.go +++ b/vendor/github.com/nwaples/rardecode/huffman.go @@ -130,7 +130,7 @@ func (h *huffmanDecoder) readSym(r bitReader) (int, error) { dist >>= maxCodeLength - bits pos := h.pos[bits] + dist - if pos > len(h.symbol) { + if pos >= len(h.symbol) { return 0, errHuffDecodeFailed } diff --git a/vendor/github.com/nwaples/rardecode/reader.go b/vendor/github.com/nwaples/rardecode/reader.go index 03e88a87..11adc4fe 100644 --- a/vendor/github.com/nwaples/rardecode/reader.go +++ b/vendor/github.com/nwaples/rardecode/reader.go @@ -356,6 +356,13 @@ func (rc *ReadCloser) Close() error { return rc.v.Close() } +// Volumes returns the volume filenames that have been used in decoding the archive +// up to this point. This will include the current open volume if the archive is still +// being processed. +func (rc *ReadCloser) Volumes() []string { + return rc.v.files +} + // OpenReader opens a RAR archive specified by the name and returns a ReadCloser. func OpenReader(name, password string) (*ReadCloser, error) { v, err := openVolume(name, password) diff --git a/vendor/github.com/palantir/godel/v2/framework/builtintasks/packages/packages.go b/vendor/github.com/palantir/godel/v2/framework/builtintasks/packages/packages.go index 32cfefbc..6597a605 100644 --- a/vendor/github.com/palantir/godel/v2/framework/builtintasks/packages/packages.go +++ b/vendor/github.com/palantir/godel/v2/framework/builtintasks/packages/packages.go @@ -21,7 +21,7 @@ import ( ) func List(exclude matcher.Matcher, wd string) ([]string, error) { - pkgs, err := pkgpath.PackagesInDir(wd, exclude) + pkgs, err := pkgpath.PackagesInDirMatchingRootModule(wd, exclude) if err != nil { return nil, errors.Wrapf(err, "failed to list packages") } diff --git a/vendor/github.com/palantir/godel/v2/framework/godellauncher/defaulttasks/defaulttasks.go b/vendor/github.com/palantir/godel/v2/framework/godellauncher/defaulttasks/defaulttasks.go index e1dbf674..8bbeb6b8 100644 --- a/vendor/github.com/palantir/godel/v2/framework/godellauncher/defaulttasks/defaulttasks.go +++ b/vendor/github.com/palantir/godel/v2/framework/godellauncher/defaulttasks/defaulttasks.go @@ -33,12 +33,12 @@ var defaultPluginsConfig = config.PluginsConfig{ { LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{ Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.distgo:dist-plugin:1.72.0", + ID: "com.palantir.distgo:dist-plugin:1.76.0", Checksums: map[string]string{ - "darwin-amd64": "17e411fd5a531adc7047f714ddd09c21a0422e9f467744f73ad5923af0981a8e", - "darwin-arm64": "5e5f8ab2c5163ebdc088a859e57ae07d1a505bb9c9dcba0865007aa1249593ed", - "linux-amd64": "e06272934464b37c59231115d987b3dc0832bfeb643f8ad49b9ab0a8e0dae577", - "linux-arm64": "a29c7d241d822a571bc519b5c355a5c29a76a9b0bfcf11b9f3b224a3d1b0023f", + "darwin-amd64": "78be7634f0f5b32d76aa7361c4a48cf53321c598beea69defd1628e8fd738382", + "darwin-arm64": "46d70c56e4fcb1fd1e747d9599b0c5ad3e70b40bcd1ba1ff4d0e973f1bd767e9", + "linux-amd64": "524d46233357b5c4f0f043aaa9822c6f735916c25289f2395dde6451d6442263", + "linux-arm64": "e46b1d6e5676a3d56ae0884b1faf915454618d233bd3b6017c577b7f71cedaf2", }, }), }), @@ -46,24 +46,24 @@ var defaultPluginsConfig = config.PluginsConfig{ { LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{ Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.godel-format-plugin:format-plugin:1.43.0", + ID: "com.palantir.godel-format-plugin:format-plugin:1.46.0", Checksums: map[string]string{ - "darwin-amd64": "cd6fb6072644fb13df9526ebe5270d1258b5b4b6ea8ece6b3db79bf1f3ec5d59", - "darwin-arm64": "32502e22cac72729a00ef1ead23cbd21d19cf5d0cda857c9ccdf68150ae5211e", - "linux-amd64": "d6e499b75d82eb50033377928e5013ccd20df673b48a13213a5ab248602812ff", - "linux-arm64": "77b486e78dcde9974318ae5843b086c235e7522a61207a43c8b1a8ce8dce317c", + "darwin-amd64": "b5210fc5e6cc2a456fb8f42e42de2622e6f216af5981708be02bdba1c0774563", + "darwin-arm64": "67f01069e73011130ed942ddee9ac3f3d6ca0eb1f8f924c48ee0f78657283b8a", + "linux-amd64": "694be1e75e492e8ba3ab1348ece542729fd1b08c6b743594ed3dc579ce1a80ab", + "linux-arm64": "e874fad3de2f51ba9a4541c61fd0d1e82919f8434c471fd4e8f1e8361cfc5043", }, }), }), Assets: config.ToLocatorWithResolverConfigs([]config.LocatorWithResolverConfig{ { Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.godel-format-asset-ptimports:ptimports-asset:1.42.0", + ID: "com.palantir.godel-format-asset-ptimports:ptimports-asset:1.45.0", Checksums: map[string]string{ - "darwin-amd64": "aa9fa7dbd5020dfaaea0a50323e3093dfacf692ed44aa56ba6753c43f5894d59", - "darwin-arm64": "4e501945aa56c9ef070899c0cb57ceb488ea2a9e84fee42cd6128641e576f154", - "linux-amd64": "1a380e6ca046cb4325cc8ed2e200df01bae3193a662635f00e1a5141d27a1c6d", - "linux-arm64": "235c21a088e890e71c7e17747bfaad25626888430826a21d797961867244aa96", + "darwin-amd64": "749c7fe7af60ce3b8308e96875d1855047bef588199679175d2fe904d529af54", + "darwin-arm64": "79928dbbe091ce3a53c38fdfa758b2205bf44b2628e2e571dc7bd6220b439c1e", + "linux-amd64": "7418e6576e0611aac590af1f75556657b1a83eafa23ce8dd36953bbd91252092", + "linux-arm64": "b988c6a4658b825a95359767e47033e1e72b7245e3d443681330a84dd33b13e1", }, }), }, @@ -72,12 +72,12 @@ var defaultPluginsConfig = config.PluginsConfig{ { LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{ Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.godel-goland-plugin:goland-plugin:1.39.0", + ID: "com.palantir.godel-goland-plugin:goland-plugin:1.43.0", Checksums: map[string]string{ - "darwin-amd64": "f0519faf444385ea41366cd37f4f55311544d3a238f5f6f24ce903136d36334c", - "darwin-arm64": "843f443ce7c90ed98d9ce6842ed396e9a2f6d626237e5ff9ddaebff18971ff25", - "linux-amd64": "65328df241c072f41f48c9b47e830a7fd1a267861fc7b8a5e36121a242a2f216", - "linux-arm64": "3542cc4d5804a456cd1a3ddaec19157e6e34032291d6b27fd7ad7feb741b3ac6", + "darwin-amd64": "7a6e85ecb8d8be9db012d58cd033cf6ec9a1f092f18aa56e6ed7756ac23552a3", + "darwin-arm64": "793ed13aab10ef0c9d48e2707e5714dc94b10765ffd000c91316d2387b427ead", + "linux-amd64": "31afde7c76bbfc58db4b1bd5b8aa24219b41b9e31411342a9a3dcb6624d34f2b", + "linux-arm64": "44afff8b5d59a89a073498685f62f3434ecff5c3664520cc71512fc1ab39ce2e", }, }), }), @@ -85,123 +85,123 @@ var defaultPluginsConfig = config.PluginsConfig{ { LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{ Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.okgo:check-plugin:1.55.0", + ID: "com.palantir.okgo:check-plugin:1.59.0", Checksums: map[string]string{ - "darwin-amd64": "e19d78628839691cb3f15879f61c55236f41cea4ab8af03aedfce4dd817aa159", - "darwin-arm64": "230a5db3b3f2477eff3b6a3a530be8b0c7d57f6de47e944407f46d21d6212fe3", - "linux-amd64": "859a75f462df56fe981dd38461bd120473bf534859d04ee0eb55a2abb2919dab", - "linux-arm64": "69dd7344729d82082646399fd92fffc7b3ae10e87cd055e3edcf98af7916ea78", + "darwin-amd64": "97d2080f8cb039f46de1474452c0abf3a4e5becb37485aca50878ecb68ad1dc8", + "darwin-arm64": "0d54796ece319fe57f8974c723796aec87cd32c5544bbd486bf1a223ec48b629", + "linux-amd64": "debfddf54935bd09db8dc0686b7d71ad85ce8af4a42a21daced36064fcf5062d", + "linux-arm64": "cbafbf938a87750162977fa37255a9bfba7aa6aa2e2e3f980d8937eac8a11201", }, }), }), Assets: config.ToLocatorWithResolverConfigs([]config.LocatorWithResolverConfig{ { Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.godel-okgo-asset-compiles:compiles-asset:1.48.0", + ID: "com.palantir.godel-okgo-asset-compiles:compiles-asset:1.51.0", Checksums: map[string]string{ - "darwin-amd64": "f04cdaf4018e89ad0dec55441d0a21f384c8a71c28e59c24bd586d3eeaaf6a91", - "darwin-arm64": "ece907c67d5cb38d7fadb96a3a0ddea4122b9e2cc60d37025b410bf4d8d3e6de", - "linux-amd64": "1bc96bc770a4edf807751e1c200a10993b5133cd3fae112a58c6279ec8d5f2a1", - "linux-arm64": "2ac5b1f6cfadcb3586db2eb3212544b22f3494b10de845b1a016d6ef729f1657", + "darwin-amd64": "6c96de0277588b420bedd4666999f0a6458bf92aebd0b35c22a88d4799db0840", + "darwin-arm64": "30fa2c3b4c2e901ebb0eb04887ead83da225fc1a14a5b5a386c8e8caca092fe9", + "linux-amd64": "725aedcf8024caf4c62cd16ee5e1b5cce504d5305439a9c6020be22d3308a328", + "linux-arm64": "e51388bd7be7afda9692c345bcb89633f50d92e3c887f66f901d0712e65d06a1", }, }), }, { Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.godel-okgo-asset-deadcode:deadcode-asset:1.44.0", + ID: "com.palantir.godel-okgo-asset-deadcode:deadcode-asset:1.45.0", Checksums: map[string]string{ - "darwin-amd64": "50a6b9efccddd903cd0fd2ce5a5fd5ffd8f2ad97c1eecdd6c2c114b5f1d9e673", - "darwin-arm64": "e922001004e5994ae4c811a23461dda014d2b3197e31787b9081c274852edb34", - "linux-amd64": "5cbab8995703205a451c78499e0751e7de608e048e343884580f01176d180501", - "linux-arm64": "0291d4f0de9c5ed8569446f33247ed37fae347b2028898c2a9f19022d50c87a7", + "darwin-amd64": "d800ad95d45aa74d8137d7c78c649b7b9fe203f89c798b69e06380537c07be5b", + "darwin-arm64": "c0ad7e7dd0437c82b98837305e7259dfd6db778260f6c8449bb548d6d90dc53a", + "linux-amd64": "41d1c816de8539bb76b76fd4a4f925469123d7fad8ce3ea4df46eea4259fa596", + "linux-arm64": "44586bcb2e6d102c6fe378bb0f6de2505e345811ca537b9c4d5e9b242c6416e6", }, }), }, { Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.godel-okgo-asset-errcheck:errcheck-asset:1.46.0", + ID: "com.palantir.godel-okgo-asset-errcheck:errcheck-asset:1.47.0", Checksums: map[string]string{ - "darwin-amd64": "d852baa3801024b571862be5f4da311ced4b78c50618a7a08de151ade071cea8", - "darwin-arm64": "4c4d8e23d2493a1c0108412ecd6c95a9a28cb0589e8106acf0ac52511f6407c7", - "linux-amd64": "ca465fec823fe9b917292b7b7496cf726e00a9d08ce57ac271df80e0e8a6c32c", - "linux-arm64": "b51593f664ab212d916141dda736a9cfdbc063fbe992f396fe19cd4a7ffa86cb", + "darwin-amd64": "71d6999d7a90a01f2c7b6107fe7fd838a0606e9ab7fb5cbe5a62198c13f374d0", + "darwin-arm64": "b0b300d5c7a578f13febbec5a6109aaf39544d306d05197a5a817ebec09eadba", + "linux-amd64": "20deb54278ab31d3ed442b0641fdc8f7e0318b27a4709e2419c9c54c2b8d0818", + "linux-arm64": "7c1d2c18cb574ade0e96998aeb69fcb5a9594be1cfee6a8b5b287b8c919d4ad6", }, }), }, { Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.godel-okgo-asset-golint:golint-asset:1.36.0", + ID: "com.palantir.godel-okgo-asset-golint:golint-asset:1.37.0", Checksums: map[string]string{ - "darwin-amd64": "4b5c75d393eac156568bbeefa5ac40254ed43d2f4f147c3f00fa6620c392c776", - "darwin-arm64": "df0060796a977ba65aa5a40e49b9e598be18f02f73e5ce54fd8e0d8093e31195", - "linux-amd64": "ca07c808072916ad71fc82bfc6d2f439488b592b8ca2097e007b0d9108e00d6b", - "linux-arm64": "1a540079e6194a1a53f4cb37b128ac46fb7437a3f74105c3fc9fb4e10677a205", + "darwin-amd64": "7e179bb625a2dc9aa0146a7d67bfa6c97073ad1123fd9d3afafa3f9a5473df97", + "darwin-arm64": "87ff384bd0b8b6fa024ea53c139fbd6efc3034cfe39835fe26b5cce683696c98", + "linux-amd64": "454405768d2e33c1a9982bf6d27c3f09612fed2c44797830ec8ce0db26813dff", + "linux-arm64": "4c81969282accb04f7656e1caa866a51fdb904aa73a689fcd8af3248c48411e3", }, }), }, { Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.godel-okgo-asset-govet:govet-asset:1.40.0", + ID: "com.palantir.godel-okgo-asset-govet:govet-asset:1.43.0", Checksums: map[string]string{ - "darwin-amd64": "db417d5eb5af7cce228a72745ab120d839f2b3e4eca8158762b36e4da6ed565d", - "darwin-arm64": "85a6e962c3373397b6a4ec2f86941753d95a28b642035422c8bca8587268458c", - "linux-amd64": "a42c031fd45b29cbde7b73bea28e61dbcc86e7aac57f373422659e0e20a854ea", - "linux-arm64": "fdfc2b48e2c7dd8009d5fc66379d4d43b93908d5c2b129d2f4871fa0786f289d", + "darwin-amd64": "179f0dcf82af53ed7539774e7388efa2dc744798a0d1127e318df5bc775e64c5", + "darwin-arm64": "631e1ecbc08a50c4c38ea9f079f1861e47b80049961d1ebe1411f67c0d25c13c", + "linux-amd64": "998fef33755658bdcb4a3fb2ca6dc9a71f13582cedf0177857add6070e440ae9", + "linux-arm64": "20f21ac0b20586fdf65a6fd52b291302d276e300db200117ae59d86c1afa0e1f", }, }), }, { Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.godel-okgo-asset-importalias:importalias-asset:1.39.0", + ID: "com.palantir.godel-okgo-asset-importalias:importalias-asset:1.42.0", Checksums: map[string]string{ - "darwin-amd64": "62f83447d5960e41371b368463c28eeaa1d1ce99f526a5ab0fc8f14d2288dd2a", - "darwin-arm64": "9dbe43492be8b66e81be9b907be8bb8fc43645678321e45e5b9069c0aae7c8c5", - "linux-amd64": "226a7c1d3371d553fb222aa810b10e35652d898a9d1e5bed25a7693b0359b608", - "linux-arm64": "b15939e9048ab525aa4c42a5487637c1c04d04e07f0b9d4e5132338c837c2ba0", + "darwin-amd64": "8e02ddbbf805ecc7e3b8a377628766816e7748771f21b83688c2778f0af59bb5", + "darwin-arm64": "d19989a6fd9e14972faffa7006ef1dcacbcf0283ea0620f666178fa0b8c8ef09", + "linux-amd64": "0cfdcfedb86479d380206a07155df6838d59a9008f28e448890b1e8104a115e9", + "linux-arm64": "9074da12d06f7e333e1adb78cd15e524773dcc55967545bdae797badf29dab20", }, }), }, { Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.godel-okgo-asset-ineffassign:ineffassign-asset:1.42.0", + ID: "com.palantir.godel-okgo-asset-ineffassign:ineffassign-asset:1.43.0", Checksums: map[string]string{ - "darwin-amd64": "8dbbd625f12b3b589d1bd8279c2d04cc66ecc7b52652a295d062ff68bd019ca0", - "darwin-arm64": "41bcd63ea66dc93be7bc20102caa91ce1255e4d472462c038865816ebe5e0d54", - "linux-amd64": "61bb90a53a11baf786297f50b70d5b1f64388013f9c54568f93234151188a0d0", - "linux-arm64": "f0a99ade6961a94b2c8dcc49b6fe069015920aee4cf9db3b6c739b1ad26d4950", + "darwin-amd64": "e12662997f3d6bc54144eb77ca8b31c5e97ace753e8f34b1c6c7804b8dc3ca56", + "darwin-arm64": "946e970b8cd1b60f02eeacc49847ec3317990a161d07b9d927ae42549ade28d8", + "linux-amd64": "7640e562ee35ad65929522386ab7245d1a51cd68a75b2b4075a3fab992d34113", + "linux-arm64": "0edcbf537dfd5a2dd7478ee3cf439e50a8dc0512aaf503af976b47446f4458d9", }, }), }, { Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.godel-okgo-asset-outparamcheck:outparamcheck-asset:1.44.0", + ID: "com.palantir.godel-okgo-asset-outparamcheck:outparamcheck-asset:1.45.0", Checksums: map[string]string{ - "darwin-amd64": "7411464066d4926e2357e69d33d1aea8bff70bd471a757b2d06fb94814898ab6", - "darwin-arm64": "62273de861d1c1b47d3144e63a3197a3e735f7d14d5f3af42e4e866d98f4a26b", - "linux-amd64": "45c154199163842e2f0eb62961b22be0161a29f89e803f9331595736cefc7943", - "linux-arm64": "55c4b4a196051a8655cd65ec1773f0f03747bad451ad3854644d6a3c69d11c58", + "darwin-amd64": "0c90072217bf31ccab8972fe0684e8e01c458673075d6b18be1fc157550b2838", + "darwin-arm64": "7b7f84628805e0585da79f88511fa1f054e622e5b5de956cd068ae1cb6b1f2d1", + "linux-amd64": "17adcd7698e90d7fe3063e3c9243586abe470e29c7a9495c4ba44e3a0117426f", + "linux-arm64": "6f988514015cfac7be674f88407867e8d970c7ec1d4446c06ca8554bfb4f42b2", }, }), }, { Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.godel-okgo-asset-unconvert:unconvert-asset:1.44.0", + ID: "com.palantir.godel-okgo-asset-unconvert:unconvert-asset:1.46.0", Checksums: map[string]string{ - "darwin-amd64": "eddf44f95ad102199463eec1955690047fc374d865b6202d6756d7209af57315", - "darwin-arm64": "476795fbf7354a99453b012302f08d9f3e5e2aaf4fe02f1be366b4ed25f72250", - "linux-amd64": "56a16bb8bb5573c7df0202731bb3302efa01bf7cb29c6bfb0d2b9270755e4bca", - "linux-arm64": "00f2bb745a3e344054003d0a0001465dfa1a7cf000f4cd6483160899c780cb56", + "darwin-amd64": "37a9200cce87723bd8c8f921898393d83f0fbe1d41970d5d10575a1883565039", + "darwin-arm64": "4ff5c078412ba52645e43dbbc8f3adf5257fbce096dfc2e60fdedc12ac245b42", + "linux-amd64": "35ad1b373f118584d186c72143c427d49e09ef4e97f4879a57f656f2109953fd", + "linux-arm64": "5be71b66d0f910fcbf8fb591a24df86990a7c175198d49027a5024559cde2339", }, }), }, { Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.godel-okgo-asset-varcheck:varcheck-asset:1.44.0", + ID: "com.palantir.godel-okgo-asset-varcheck:varcheck-asset:1.45.0", Checksums: map[string]string{ - "darwin-amd64": "4016bba7b72598e46e898ab17e8a1a891b760f229b494c3e89a13b6c932c931e", - "darwin-arm64": "16dcb961026dcd6ec0a96980db3797bd0f260e64ffd89832c400df06394a4313", - "linux-amd64": "53a832512b31d58408c6a06378e17a10c51382d53dca585c7c139637b8f41542", - "linux-arm64": "e977325bc418ce9b48da189950ea425d0e14528c7e99de071e81621c7798089a", + "darwin-amd64": "da77d28095d5355efc80701489afe7388248069fb00a9574a5a066aaf741ef62", + "darwin-arm64": "45a4bcb158284ad6fdacc6d0acdbc530b579e54755bb264544b1014ea9621ca4", + "linux-amd64": "abcb7cd750debcea5230307b0935cb850a6308f4ae316df006ac9dc2bd828713", + "linux-arm64": "ce010e1cc472efe891228457e0d086ea8ad26a6eb2b5b1fbb3f660ca5ccc25cc", }, }), }, @@ -210,12 +210,12 @@ var defaultPluginsConfig = config.PluginsConfig{ { LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{ Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.godel-license-plugin:license-plugin:1.42.0", + ID: "com.palantir.godel-license-plugin:license-plugin:1.45.0", Checksums: map[string]string{ - "darwin-amd64": "1b45b215ffbf4c202ee0feca039fff1fe2faf99151d393c30906d0c7e4356f51", - "darwin-arm64": "2a89fc240ca55f2c9d37ceb3b1fe38ef0d9ed4dcc59838fdd02de3ce2cba2cd0", - "linux-amd64": "7abac4848e74d92fa341e971a3242d25bab15eea088ffc784f3490aa45871bb6", - "linux-arm64": "330c327913b937f404c49ec81f7f60e2b2fa54d40620170332c337ff11f84cd0", + "darwin-amd64": "f789038f283664077009d3d291b5a2e00e434401c9d0ec0ebdb2a60a6d574a99", + "darwin-arm64": "907dfcae5a8f96eb3edc9277e02311da33e4845a0d270b00b86aecdc6a28a349", + "linux-amd64": "0dd92fd3f74c3d7c48be1cbf86de2b885535a8a784a72a383ca0bfec9517f212", + "linux-arm64": "a3dd2fd1bbaa9304d8b3d11a141b418400cd4ff00522e09249d22853ee787efe", }, }), }), @@ -223,12 +223,12 @@ var defaultPluginsConfig = config.PluginsConfig{ { LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{ Locator: config.ToLocatorConfig(config.LocatorConfig{ - ID: "com.palantir.godel-test-plugin:test-plugin:1.41.0", + ID: "com.palantir.godel-test-plugin:test-plugin:1.42.0", Checksums: map[string]string{ - "darwin-amd64": "a8d8ed537fb632bc80c1c13e16e240e41461a32bb50a1902cfc06df1b10d019e", - "darwin-arm64": "e93a58a510bd16f19bc223e242334fc0ded842f6ce813b209b028962539e7a8c", - "linux-amd64": "461ba23c82cb7a98392b20e87fdca8efbdbb4af0178a2f71a0f485a9bbafc647", - "linux-arm64": "56b1433a9529791834ccb736550f9b70e0928d72774c5a9c0848c7c92a333025", + "darwin-amd64": "cc7c3294e6dfd32f4f116f1f91b560dadb599b20e4a879ee4e73fa1bdbf3ca5e", + "darwin-arm64": "0d373b10743095e90a52fa17ebe3f35cd79ca24709fca1f599eb93fe1b925912", + "linux-amd64": "f1be4fbfb707a61a7a6fad6cea0cec00db76c358e08851c25758fd115c030d47", + "linux-arm64": "512cde17fbba7070bd1dacc01a52ed72c0495c46c765695d53c7a4a0298b5e42", }, }), }), diff --git a/vendor/github.com/palantir/pkg/pkgpath/godelw b/vendor/github.com/palantir/pkg/pkgpath/godelw index db902759..4c672faa 100644 --- a/vendor/github.com/palantir/pkg/pkgpath/godelw +++ b/vendor/github.com/palantir/pkg/pkgpath/godelw @@ -3,11 +3,11 @@ set -euo pipefail # Version and checksums for godel. Values are populated by the godel "dist" task. -VERSION=2.67.0 -DARWIN_AMD64_CHECKSUM=323edc3ba73150e46e59a90146e086e5587b1d402f3288dd79e43ff856907e37 -DARWIN_ARM64_CHECKSUM=528db442c313060abdacf2bb496d4adb49dcf492924e8b62c0f699f25aeed358 -LINUX_AMD64_CHECKSUM=b58be2904fd04e2238af0ddfa1021000e2b0e9f9d530c04b3ea66e3f0fcc47b5 -LINUX_ARM64_CHECKSUM=dd96d9a622dc28054cd6d3f83322ecf53a02ff481500b0d0f6d3a757685df13f +VERSION=2.112.0 +DARWIN_AMD64_CHECKSUM=db7ea4b241723b33acb0c82d1113b6ef8e2a4d499ad6f4bd749ad9725c5bc5a2 +DARWIN_ARM64_CHECKSUM=94a5d8825c9998f0c5f5b3622cad60e6c298411b30b3ece4d0fe9222c0a0416f +LINUX_AMD64_CHECKSUM=3516173db9bc3ba00f325474000a027a1f8bb9fe83afa4916be56b98ad53fcf8 +LINUX_ARM64_CHECKSUM=eeff17150fc12d32a1ac8a6b36b7a12a87fa4f2cf47f59024ac7e8f70451e0a6 # Downloads file at URL to destination path using wget or curl. Prints an error and exits if wget or curl is not present. function download { diff --git a/vendor/github.com/palantir/pkg/pkgpath/packages.go b/vendor/github.com/palantir/pkg/pkgpath/packages.go index 66b962b0..befe3255 100644 --- a/vendor/github.com/palantir/pkg/pkgpath/packages.go +++ b/vendor/github.com/palantir/pkg/pkgpath/packages.go @@ -9,6 +9,7 @@ import ( "go/build" "go/parser" "go/token" + "io/fs" "os" "path" "path/filepath" @@ -16,6 +17,8 @@ import ( "strings" "github.com/palantir/pkg/matcher" + "golang.org/x/mod/modfile" + gopackages "golang.org/x/tools/go/packages" ) // DefaultGoPkgExcludeMatcher returns a matcher that matches names that standard Go tools generally exclude as Go @@ -228,36 +231,80 @@ func PackagesFromPaths(rootDir string, relPaths []string) (Packages, error) { // PackagesInDir creates a Packages that contains all of the packages rooted at the provided directory. Every directory // rooted in the provided directory whose path does not match the provided exclude matcher is considered as a package. func PackagesInDir(rootDir string, exclude matcher.Matcher) (Packages, error) { + return packagesInDir(rootDir, nil, exclude) +} + +// PackagesInDirMatchingRootModule creates a Packages that contains all of the packages rooted at the provided directory +// that are part of the same module as the root directory. Every directory rooted in the provided directory whose path +// does not match the provided exclude matcher and is part of the same module as the module of the root directory is +// considered as a package. +func PackagesInDirMatchingRootModule(rootDir string, exclude matcher.Matcher) (Packages, error) { + return packagesInDir(rootDir, nonRootModuleExcluder(rootDir), exclude) +} + +// packagesInDir creates a Packages that contains all of the packages rooted at the provided directory. Every +// directory rooted in the provided directory whose path does not match the provided exclude matcher is considered as a +// package. A directory is only considered if pkgDirExcluder and pkgFileExcluder do not match it. If a directory is +// considered, its package will be determined based on considering only the files in the directory that do not match the +// pkgFileExcluder. If pkgDirExcluder returns a value that indicates that the subdirectories should be skipped, those +// subdirectories will not be considered. +func packagesInDir(rootDir string, pkgDirExcluder pkgExcluder, pkgFileExcluder matcher.Matcher) (Packages, error) { dirAbsolutePath, err := filepath.Abs(rootDir) if err != nil { return nil, fmt.Errorf("failed to convert %s to absolute path: %v", rootDir, err) } allPkgs := make(map[string]string) - if err := filepath.Walk(dirAbsolutePath, func(currPath string, currInfo os.FileInfo, err error) error { - currRelPath, currRelPathErr := filepath.Rel(dirAbsolutePath, currPath) - - // skip current path if it matches an exclude - if currRelPathErr == nil && exclude != nil && exclude.Match(currRelPath) { - return nil - } - + if err := filepath.WalkDir(dirAbsolutePath, func(currPath string, currInfo fs.DirEntry, err error) error { + // if there was any error reading path, return error if err != nil { return err } + // skip path if it is not a directory if !currInfo.IsDir() { return nil } + // determine relative path for package + currRelPath, currRelPathErr := filepath.Rel(dirAbsolutePath, currPath) if currRelPathErr != nil { return currRelPathErr } + skipDir := false + + // if pkgDirExcluder is non-nil, check if directory should be excluded + if pkgDirExcluder != nil { + // determine whether current directory should be excluded + excludeDir, skipAllSubDirs := pkgDirExcluder.Exclude(currRelPath) + + // update value of "skipDir" parameter: this will be used later (even if current directory is not excluded, + // if this value is true, the subdirectories will still be skipped). + skipDir = skipAllSubDirs + + // if current directory should be excluded, return. Use "skipDir" parameter to determine whether all + // subdirectories should be skipped. + if excludeDir { + if skipDir { + return filepath.SkipDir + } + return nil + } + } + + // if pkgFileExcluder is non-nil, check if directory should be excluded + if pkgFileExcluder != nil && pkgFileExcluder.Match(currRelPath) { + if skipDir { + return filepath.SkipDir + } + return nil + } + // create a filter for processing package files that only passes if it does not match an exclude filter := func(info os.FileInfo) bool { // if exclude exists and matches the file, skip it - if exclude != nil && exclude.Match(path.Join(currRelPath, info.Name())) { + if pkgFileExcluder != nil && pkgFileExcluder.Match(path.Join(currRelPath, info.Name())) { return false } // process file if it would be included in build context (handles things like build tags) @@ -274,6 +321,12 @@ func PackagesInDir(rootDir string, exclude matcher.Matcher) (Packages, error) { allPkgs[currPath] = pkgName } + // if this directory matched but return value indicates that subdirectories should not be considered, return + // filepath.SkipDir + if skipDir { + return filepath.SkipDir + } + return nil }); err != nil { return nil, err @@ -282,6 +335,63 @@ func PackagesInDir(rootDir string, exclude matcher.Matcher) (Packages, error) { return createPkgsWithValidation(dirAbsolutePath, allPkgs) } +func nonRootModuleExcluder(wd string) pkgExcluder { + wdPkgs, err := gopackages.Load(&gopackages.Config{ + Mode: gopackages.NeedModule, + Dir: wd, + }, ".") + if err != nil || len(wdPkgs) == 0 || wdPkgs[0].Module == nil || wdPkgs[0].Module.Path == "" { + return nil + } + + wdModulePath := wdPkgs[0].Module.Path + return pkgExcluderFn(func(relPath string) (exclude, excludeAllSubdirs bool) { + fullPath := filepath.Join(wd, relPath) + dirEntries, readDirErr := os.ReadDir(fullPath) + // path cannot be read: do not exclude + if readDirErr != nil { + return false, false + } + var goModFilePath string + for _, entry := range dirEntries { + // skip all entries that are not the "go.mod" file + if entry.IsDir() || entry.Name() != "go.mod" { + continue + } + goModFilePath = filepath.Join(fullPath, entry.Name()) + break + } + // no "go.mod" file in directory: do not exclude + if goModFilePath == "" { + return false, false + } + + // "go.mod" file exists in directory + modFileBytes, err := os.ReadFile(goModFilePath) + + // "go.mod" file cannot be read: do not exclude + if err != nil { + return false, false + } + // if module path of directory does not match root module, exclude this directory and all subdirectories + differentModule := modfile.ModulePath(modFileBytes) != wdModulePath + return differentModule, differentModule + }) +} + +type pkgExcluder interface { + // Exclude returns true if the package at the specified path should be excluded when matching packages. The relpath + // parameter will always be a directory. If the second return value is true, then no subdirectories of the package + // directories will be considered. + Exclude(relPath string) (exclude, excludeAllSubdirs bool) +} + +type pkgExcluderFn func(relPath string) (exclude, excludeAllSubdirs bool) + +func (fn pkgExcluderFn) Exclude(relPath string) (exclude, excludeAllSubdirs bool) { + return fn(relPath) +} + func createPkgsWithValidation(rootDir string, pkgs map[string]string) (*packages, error) { if !path.IsAbs(rootDir) { return nil, fmt.Errorf("rootDir %s is not an absolute path", rootDir) diff --git a/vendor/github.com/pierrec/lz4/.travis.yml b/vendor/github.com/pierrec/lz4/.travis.yml deleted file mode 100644 index b2c806d5..00000000 --- a/vendor/github.com/pierrec/lz4/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -language: go - -go: - - 1.8.x - - 1.9.x - - 1.10.x - - master - -matrix: - fast_finish: true - allow_failures: - - go: master - -sudo: false - -script: - - go test -v -cpu=2 - - go test -v -cpu=2 -race diff --git a/vendor/github.com/pierrec/lz4/README.md b/vendor/github.com/pierrec/lz4/README.md deleted file mode 100644 index 50a10ee1..00000000 --- a/vendor/github.com/pierrec/lz4/README.md +++ /dev/null @@ -1,24 +0,0 @@ -[![godoc](https://godoc.org/github.com/pierrec/lz4?status.png)](https://godoc.org/github.com/pierrec/lz4) - -# lz4 -LZ4 compression and decompression in pure Go. - -## Usage - -```go -import "github.com/pierrec/lz4" -``` - -## Description -Package lz4 implements reading and writing lz4 compressed data (a frame), -as specified in http://fastcompression.blogspot.fr/2013/04/lz4-streaming-format-final.html. - -This package is **compatible with the LZ4 frame format** although the block level compression -and decompression functions are exposed and are fully compatible with the lz4 block format -definition, they are low level and should not be used directly. - -For a complete description of an lz4 compressed block, see: -http://fastcompression.blogspot.fr/2011/05/lz4-explained.html - -See https://github.com/Cyan4973/lz4 for the reference C implementation. - diff --git a/vendor/github.com/pierrec/lz4/block.go b/vendor/github.com/pierrec/lz4/block.go deleted file mode 100644 index ef24f17e..00000000 --- a/vendor/github.com/pierrec/lz4/block.go +++ /dev/null @@ -1,397 +0,0 @@ -package lz4 - -import ( - "encoding/binary" - "errors" -) - -var ( - // ErrInvalidSourceShortBuffer is returned by UncompressBlock or CompressBLock when a compressed - // block is corrupted or the destination buffer is not large enough for the uncompressed data. - ErrInvalidSourceShortBuffer = errors.New("lz4: invalid source or destination buffer too short") - // ErrInvalid is returned when reading an invalid LZ4 archive. - ErrInvalid = errors.New("lz4: bad magic number") -) - -// blockHash hashes 4 bytes into a value < winSize. -func blockHash(x uint32) uint32 { - const hasher uint32 = 2654435761 // Knuth multiplicative hash. - return x * hasher >> hashShift -} - -// CompressBlockBound returns the maximum size of a given buffer of size n, when not compressible. -func CompressBlockBound(n int) int { - return n + n/255 + 16 -} - -// UncompressBlock uncompresses the source buffer into the destination one, -// and returns the uncompressed size. -// -// The destination buffer must be sized appropriately. -// -// An error is returned if the source data is invalid or the destination buffer is too small. -func UncompressBlock(src, dst []byte) (si int, err error) { - defer func() { - // It is now faster to let the runtime panic and recover on out of bound slice access - // than checking indices as we go along. - if recover() != nil { - err = ErrInvalidSourceShortBuffer - } - }() - sn := len(src) - if sn == 0 { - return 0, nil - } - var di int - - for { - // Literals and match lengths (token). - b := int(src[si]) - si++ - - // Literals. - if lLen := b >> 4; lLen > 0 { - if lLen == 0xF { - for src[si] == 0xFF { - lLen += 0xFF - si++ - } - lLen += int(src[si]) - si++ - } - i := si - si += lLen - di += copy(dst[di:], src[i:si]) - - if si >= sn { - return di, nil - } - } - - si++ - _ = src[si] // Bound check elimination. - offset := int(src[si-1]) | int(src[si])<<8 - si++ - - // Match. - mLen := b & 0xF - if mLen == 0xF { - for src[si] == 0xFF { - mLen += 0xFF - si++ - } - mLen += int(src[si]) - si++ - } - mLen += minMatch - - // Copy the match. - i := di - offset - if offset > 0 && mLen >= offset { - // Efficiently copy the match dst[di-offset:di] into the dst slice. - bytesToCopy := offset * (mLen / offset) - expanded := dst[i:] - for n := offset; n <= bytesToCopy+offset; n *= 2 { - copy(expanded[n:], expanded[:n]) - } - di += bytesToCopy - mLen -= bytesToCopy - } - di += copy(dst[di:], dst[i:i+mLen]) - } -} - -// CompressBlock compresses the source buffer into the destination one. -// This is the fast version of LZ4 compression and also the default one. -// The size of hashTable must be at least 64Kb. -// -// The size of the compressed data is returned. If it is 0 and no error, then the data is incompressible. -// -// An error is returned if the destination buffer is too small. -func CompressBlock(src, dst []byte, hashTable []int) (di int, err error) { - defer func() { - if recover() != nil { - err = ErrInvalidSourceShortBuffer - } - }() - - sn, dn := len(src)-mfLimit, len(dst) - if sn <= 0 || dn == 0 { - return 0, nil - } - var si int - - // Fast scan strategy: the hash table only stores the last 4 bytes sequences. - // const accInit = 1 << skipStrength - - anchor := si // Position of the current literals. - // acc := accInit // Variable step: improves performance on non-compressible data. - - for si < sn { - // Hash the next 4 bytes (sequence)... - match := binary.LittleEndian.Uint32(src[si:]) - h := blockHash(match) - - ref := hashTable[h] - hashTable[h] = si - if ref >= sn { // Invalid reference (dirty hashtable). - si++ - continue - } - offset := si - ref - if offset <= 0 || offset >= winSize || // Out of window. - match != binary.LittleEndian.Uint32(src[ref:]) { // Hash collision on different matches. - // si += acc >> skipStrength - // acc++ - si++ - continue - } - - // Match found. - // acc = accInit - lLen := si - anchor // Literal length. - - // Encode match length part 1. - si += minMatch - mLen := si // Match length has minMatch already. - // Find the longest match, first looking by batches of 8 bytes. - for si < sn && binary.LittleEndian.Uint64(src[si:]) == binary.LittleEndian.Uint64(src[si-offset:]) { - si += 8 - } - // Then byte by byte. - for si < sn && src[si] == src[si-offset] { - si++ - } - - mLen = si - mLen - if mLen < 0xF { - dst[di] = byte(mLen) - } else { - dst[di] = 0xF - } - - // Encode literals length. - if lLen < 0xF { - dst[di] |= byte(lLen << 4) - } else { - dst[di] |= 0xF0 - di++ - l := lLen - 0xF - for ; l >= 0xFF; l -= 0xFF { - dst[di] = 0xFF - di++ - } - dst[di] = byte(l) - } - di++ - - // Literals. - copy(dst[di:], src[anchor:anchor+lLen]) - di += lLen + 2 - anchor = si - - // Encode offset. - _ = dst[di] // Bound check elimination. - dst[di-2], dst[di-1] = byte(offset), byte(offset>>8) - - // Encode match length part 2. - if mLen >= 0xF { - for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF { - dst[di] = 0xFF - di++ - } - dst[di] = byte(mLen) - di++ - } - } - - if anchor == 0 { - // Incompressible. - return 0, nil - } - - // Last literals. - lLen := len(src) - anchor - if lLen < 0xF { - dst[di] = byte(lLen << 4) - } else { - dst[di] = 0xF0 - di++ - for lLen -= 0xF; lLen >= 0xFF; lLen -= 0xFF { - dst[di] = 0xFF - di++ - } - dst[di] = byte(lLen) - } - di++ - - // Write the last literals. - if di >= anchor { - // Incompressible. - return 0, nil - } - di += copy(dst[di:], src[anchor:]) - return di, nil -} - -// CompressBlockHC compresses the source buffer src into the destination dst -// with max search depth (use 0 or negative value for no max). -// -// CompressBlockHC compression ratio is better than CompressBlock but it is also slower. -// -// The size of the compressed data is returned. If it is 0 and no error, then the data is not compressible. -// -// An error is returned if the destination buffer is too small. -func CompressBlockHC(src, dst []byte, depth int) (di int, err error) { - defer func() { - if recover() != nil { - err = ErrInvalidSourceShortBuffer - } - }() - - sn, dn := len(src)-mfLimit, len(dst) - if sn <= 0 || dn == 0 { - return 0, nil - } - var si int - - // hashTable: stores the last position found for a given hash - // chaingTable: stores previous positions for a given hash - var hashTable, chainTable [winSize]int - - if depth <= 0 { - depth = winSize - } - - anchor := si - for si < sn { - // Hash the next 4 bytes (sequence). - match := binary.LittleEndian.Uint32(src[si:]) - h := blockHash(match) - - // Follow the chain until out of window and give the longest match. - mLen := 0 - offset := 0 - for next, try := hashTable[h], depth; try > 0 && next > 0 && si-next < winSize; next = chainTable[next&winMask] { - // The first (mLen==0) or next byte (mLen>=minMatch) at current match length - // must match to improve on the match length. - if src[next+mLen] != src[si+mLen] { - continue - } - ml := 0 - // Compare the current position with a previous with the same hash. - for ml < sn-si && binary.LittleEndian.Uint64(src[next+ml:]) == binary.LittleEndian.Uint64(src[si+ml:]) { - ml += 8 - } - for ml < sn-si && src[next+ml] == src[si+ml] { - ml++ - } - if ml+1 < minMatch || ml <= mLen { - // Match too small ( winStart { - winStart = ws - } - for si, ml := winStart, si+mLen; si < ml; { - match >>= 8 - match |= uint32(src[si+3]) << 24 - h := blockHash(match) - chainTable[si&winMask] = hashTable[h] - hashTable[h] = si - si++ - } - - lLen := si - anchor - si += mLen - mLen -= minMatch // Match length does not include minMatch. - - if mLen < 0xF { - dst[di] = byte(mLen) - } else { - dst[di] = 0xF - } - - // Encode literals length. - if lLen < 0xF { - dst[di] |= byte(lLen << 4) - } else { - dst[di] |= 0xF0 - di++ - l := lLen - 0xF - for ; l >= 0xFF; l -= 0xFF { - dst[di] = 0xFF - di++ - } - dst[di] = byte(l) - } - di++ - - // Literals. - copy(dst[di:], src[anchor:anchor+lLen]) - di += lLen - anchor = si - - // Encode offset. - di += 2 - dst[di-2], dst[di-1] = byte(offset), byte(offset>>8) - - // Encode match length part 2. - if mLen >= 0xF { - for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF { - dst[di] = 0xFF - di++ - } - dst[di] = byte(mLen) - di++ - } - } - - if anchor == 0 { - // Incompressible. - return 0, nil - } - - // Last literals. - lLen := len(src) - anchor - if lLen < 0xF { - dst[di] = byte(lLen << 4) - } else { - dst[di] = 0xF0 - di++ - lLen -= 0xF - for ; lLen >= 0xFF; lLen -= 0xFF { - dst[di] = 0xFF - di++ - } - dst[di] = byte(lLen) - } - di++ - - // Write the last literals. - if di >= anchor { - // Incompressible. - return 0, nil - } - di += copy(dst[di:], src[anchor:]) - return di, nil -} diff --git a/vendor/github.com/pierrec/lz4/debug.go b/vendor/github.com/pierrec/lz4/debug.go deleted file mode 100644 index bc5e78d4..00000000 --- a/vendor/github.com/pierrec/lz4/debug.go +++ /dev/null @@ -1,23 +0,0 @@ -// +build lz4debug - -package lz4 - -import ( - "fmt" - "os" - "path/filepath" - "runtime" -) - -const debugFlag = true - -func debug(args ...interface{}) { - _, file, line, _ := runtime.Caller(1) - file = filepath.Base(file) - - f := fmt.Sprintf("LZ4: %s:%d %s", file, line, args[0]) - if f[len(f)-1] != '\n' { - f += "\n" - } - fmt.Fprintf(os.Stderr, f, args[1:]...) -} diff --git a/vendor/github.com/pierrec/lz4/debug_stub.go b/vendor/github.com/pierrec/lz4/debug_stub.go deleted file mode 100644 index 44211ad9..00000000 --- a/vendor/github.com/pierrec/lz4/debug_stub.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build !lz4debug - -package lz4 - -const debugFlag = false - -func debug(args ...interface{}) {} diff --git a/vendor/github.com/pierrec/lz4/internal/xxh32/xxh32zero.go b/vendor/github.com/pierrec/lz4/internal/xxh32/xxh32zero.go deleted file mode 100644 index 850a6fdf..00000000 --- a/vendor/github.com/pierrec/lz4/internal/xxh32/xxh32zero.go +++ /dev/null @@ -1,222 +0,0 @@ -// Package xxh32 implements the very fast XXH hashing algorithm (32 bits version). -// (https://github.com/Cyan4973/XXH/) -package xxh32 - -import ( - "encoding/binary" -) - -const ( - prime32_1 uint32 = 2654435761 - prime32_2 uint32 = 2246822519 - prime32_3 uint32 = 3266489917 - prime32_4 uint32 = 668265263 - prime32_5 uint32 = 374761393 - - prime32_1plus2 uint32 = 606290984 - prime32_minus1 uint32 = 1640531535 -) - -// XXHZero represents an xxhash32 object with seed 0. -type XXHZero struct { - v1 uint32 - v2 uint32 - v3 uint32 - v4 uint32 - totalLen uint64 - buf [16]byte - bufused int -} - -// Sum appends the current hash to b and returns the resulting slice. -// It does not change the underlying hash state. -func (xxh XXHZero) Sum(b []byte) []byte { - h32 := xxh.Sum32() - return append(b, byte(h32), byte(h32>>8), byte(h32>>16), byte(h32>>24)) -} - -// Reset resets the Hash to its initial state. -func (xxh *XXHZero) Reset() { - xxh.v1 = prime32_1plus2 - xxh.v2 = prime32_2 - xxh.v3 = 0 - xxh.v4 = prime32_minus1 - xxh.totalLen = 0 - xxh.bufused = 0 -} - -// Size returns the number of bytes returned by Sum(). -func (xxh *XXHZero) Size() int { - return 4 -} - -// BlockSize gives the minimum number of bytes accepted by Write(). -func (xxh *XXHZero) BlockSize() int { - return 1 -} - -// Write adds input bytes to the Hash. -// It never returns an error. -func (xxh *XXHZero) Write(input []byte) (int, error) { - if xxh.totalLen == 0 { - xxh.Reset() - } - n := len(input) - m := xxh.bufused - - xxh.totalLen += uint64(n) - - r := len(xxh.buf) - m - if n < r { - copy(xxh.buf[m:], input) - xxh.bufused += len(input) - return n, nil - } - - p := 0 - // Causes compiler to work directly from registers instead of stack: - v1, v2, v3, v4 := xxh.v1, xxh.v2, xxh.v3, xxh.v4 - if m > 0 { - // some data left from previous update - copy(xxh.buf[xxh.bufused:], input[:r]) - xxh.bufused += len(input) - r - - // fast rotl(13) - buf := xxh.buf[:16] // BCE hint. - v1 = rol13(v1+binary.LittleEndian.Uint32(buf[:])*prime32_2) * prime32_1 - v2 = rol13(v2+binary.LittleEndian.Uint32(buf[4:])*prime32_2) * prime32_1 - v3 = rol13(v3+binary.LittleEndian.Uint32(buf[8:])*prime32_2) * prime32_1 - v4 = rol13(v4+binary.LittleEndian.Uint32(buf[12:])*prime32_2) * prime32_1 - p = r - xxh.bufused = 0 - } - - for n := n - 16; p <= n; p += 16 { - sub := input[p:][:16] //BCE hint for compiler - v1 = rol13(v1+binary.LittleEndian.Uint32(sub[:])*prime32_2) * prime32_1 - v2 = rol13(v2+binary.LittleEndian.Uint32(sub[4:])*prime32_2) * prime32_1 - v3 = rol13(v3+binary.LittleEndian.Uint32(sub[8:])*prime32_2) * prime32_1 - v4 = rol13(v4+binary.LittleEndian.Uint32(sub[12:])*prime32_2) * prime32_1 - } - xxh.v1, xxh.v2, xxh.v3, xxh.v4 = v1, v2, v3, v4 - - copy(xxh.buf[xxh.bufused:], input[p:]) - xxh.bufused += len(input) - p - - return n, nil -} - -// Sum32 returns the 32 bits Hash value. -func (xxh *XXHZero) Sum32() uint32 { - h32 := uint32(xxh.totalLen) - if h32 >= 16 { - h32 += rol1(xxh.v1) + rol7(xxh.v2) + rol12(xxh.v3) + rol18(xxh.v4) - } else { - h32 += prime32_5 - } - - p := 0 - n := xxh.bufused - buf := xxh.buf - for n := n - 4; p <= n; p += 4 { - h32 += binary.LittleEndian.Uint32(buf[p:p+4]) * prime32_3 - h32 = rol17(h32) * prime32_4 - } - for ; p < n; p++ { - h32 += uint32(buf[p]) * prime32_5 - h32 = rol11(h32) * prime32_1 - } - - h32 ^= h32 >> 15 - h32 *= prime32_2 - h32 ^= h32 >> 13 - h32 *= prime32_3 - h32 ^= h32 >> 16 - - return h32 -} - -// ChecksumZero returns the 32bits Hash value. -func ChecksumZero(input []byte) uint32 { - n := len(input) - h32 := uint32(n) - - if n < 16 { - h32 += prime32_5 - } else { - v1 := prime32_1plus2 - v2 := prime32_2 - v3 := uint32(0) - v4 := prime32_minus1 - p := 0 - for n := n - 16; p <= n; p += 16 { - sub := input[p:][:16] //BCE hint for compiler - v1 = rol13(v1+binary.LittleEndian.Uint32(sub[:])*prime32_2) * prime32_1 - v2 = rol13(v2+binary.LittleEndian.Uint32(sub[4:])*prime32_2) * prime32_1 - v3 = rol13(v3+binary.LittleEndian.Uint32(sub[8:])*prime32_2) * prime32_1 - v4 = rol13(v4+binary.LittleEndian.Uint32(sub[12:])*prime32_2) * prime32_1 - } - input = input[p:] - n -= p - h32 += rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) - } - - p := 0 - for n := n - 4; p <= n; p += 4 { - h32 += binary.LittleEndian.Uint32(input[p:p+4]) * prime32_3 - h32 = rol17(h32) * prime32_4 - } - for p < n { - h32 += uint32(input[p]) * prime32_5 - h32 = rol11(h32) * prime32_1 - p++ - } - - h32 ^= h32 >> 15 - h32 *= prime32_2 - h32 ^= h32 >> 13 - h32 *= prime32_3 - h32 ^= h32 >> 16 - - return h32 -} - -// Uint32Zero hashes x with seed 0. -func Uint32Zero(x uint32) uint32 { - h := prime32_5 + 4 + x*prime32_3 - h = rol17(h) * prime32_4 - h ^= h >> 15 - h *= prime32_2 - h ^= h >> 13 - h *= prime32_3 - h ^= h >> 16 - return h -} - -func rol1(u uint32) uint32 { - return u<<1 | u>>31 -} - -func rol7(u uint32) uint32 { - return u<<7 | u>>25 -} - -func rol11(u uint32) uint32 { - return u<<11 | u>>21 -} - -func rol12(u uint32) uint32 { - return u<<12 | u>>20 -} - -func rol13(u uint32) uint32 { - return u<<13 | u>>19 -} - -func rol17(u uint32) uint32 { - return u<<17 | u>>15 -} - -func rol18(u uint32) uint32 { - return u<<18 | u>>14 -} diff --git a/vendor/github.com/pierrec/lz4/lz4.go b/vendor/github.com/pierrec/lz4/lz4.go deleted file mode 100644 index 35802756..00000000 --- a/vendor/github.com/pierrec/lz4/lz4.go +++ /dev/null @@ -1,68 +0,0 @@ -// Package lz4 implements reading and writing lz4 compressed data (a frame), -// as specified in http://fastcompression.blogspot.fr/2013/04/lz4-streaming-format-final.html. -// -// Although the block level compression and decompression functions are exposed and are fully compatible -// with the lz4 block format definition, they are low level and should not be used directly. -// For a complete description of an lz4 compressed block, see: -// http://fastcompression.blogspot.fr/2011/05/lz4-explained.html -// -// See https://github.com/Cyan4973/lz4 for the reference C implementation. -// -package lz4 - -const ( - // Extension is the LZ4 frame file name extension - Extension = ".lz4" - // Version is the LZ4 frame format version - Version = 1 - - frameMagic uint32 = 0x184D2204 - frameSkipMagic uint32 = 0x184D2A50 - - // The following constants are used to setup the compression algorithm. - minMatch = 4 // the minimum size of the match sequence size (4 bytes) - winSizeLog = 16 // LZ4 64Kb window size limit - winSize = 1 << winSizeLog - winMask = winSize - 1 // 64Kb window of previous data for dependent blocks - compressedBlockFlag = 1 << 31 - compressedBlockMask = compressedBlockFlag - 1 - - // hashLog determines the size of the hash table used to quickly find a previous match position. - // Its value influences the compression speed and memory usage, the lower the faster, - // but at the expense of the compression ratio. - // 16 seems to be the best compromise. - hashLog = 16 - hashTableSize = 1 << hashLog - hashShift = uint((minMatch * 8) - hashLog) - - mfLimit = 8 + minMatch // The last match cannot start within the last 12 bytes. - skipStrength = 6 // variable step for fast scan -) - -// map the block max size id with its value in bytes: 64Kb, 256Kb, 1Mb and 4Mb. -var ( - bsMapID = map[byte]int{4: 64 << 10, 5: 256 << 10, 6: 1 << 20, 7: 4 << 20} - bsMapValue = make(map[int]byte, len(bsMapID)) -) - -// Reversed. -func init() { - for i, v := range bsMapID { - bsMapValue[v] = i - } -} - -// Header describes the various flags that can be set on a Writer or obtained from a Reader. -// The default values match those of the LZ4 frame format definition -// (http://fastcompression.blogspot.com/2013/04/lz4-streaming-format-final.html). -// -// NB. in a Reader, in case of concatenated frames, the Header values may change between Read() calls. -// It is the caller responsibility to check them if necessary. -type Header struct { - BlockChecksum bool // Compressed blocks checksum flag. - NoChecksum bool // Frame checksum flag. - BlockMaxSize int // Size of the uncompressed data block (one of [64KB, 256KB, 1MB, 4MB]). Default=4MB. - Size uint64 // Frame total size. It is _not_ computed by the Writer. - CompressionLevel int // Compression level (higher is better, use 0 for fastest compression). - done bool // Header processed flag (Read or Write and checked). -} diff --git a/vendor/github.com/pierrec/lz4/lz4_go1.10.go b/vendor/github.com/pierrec/lz4/lz4_go1.10.go deleted file mode 100644 index 9a0fb007..00000000 --- a/vendor/github.com/pierrec/lz4/lz4_go1.10.go +++ /dev/null @@ -1,29 +0,0 @@ -//+build go1.10 - -package lz4 - -import ( - "fmt" - "strings" -) - -func (h Header) String() string { - var s strings.Builder - - s.WriteString(fmt.Sprintf("%T{", h)) - if h.BlockChecksum { - s.WriteString("BlockChecksum: true ") - } - if h.NoChecksum { - s.WriteString("NoChecksum: true ") - } - if bs := h.BlockMaxSize; bs != 0 && bs != 4<<20 { - s.WriteString(fmt.Sprintf("BlockMaxSize: %d ", bs)) - } - if l := h.CompressionLevel; l != 0 { - s.WriteString(fmt.Sprintf("CompressionLevel: %d ", l)) - } - s.WriteByte('}') - - return s.String() -} diff --git a/vendor/github.com/pierrec/lz4/lz4_notgo1.10.go b/vendor/github.com/pierrec/lz4/lz4_notgo1.10.go deleted file mode 100644 index 12c761a2..00000000 --- a/vendor/github.com/pierrec/lz4/lz4_notgo1.10.go +++ /dev/null @@ -1,29 +0,0 @@ -//+build !go1.10 - -package lz4 - -import ( - "bytes" - "fmt" -) - -func (h Header) String() string { - var s bytes.Buffer - - s.WriteString(fmt.Sprintf("%T{", h)) - if h.BlockChecksum { - s.WriteString("BlockChecksum: true ") - } - if h.NoChecksum { - s.WriteString("NoChecksum: true ") - } - if bs := h.BlockMaxSize; bs != 0 && bs != 4<<20 { - s.WriteString(fmt.Sprintf("BlockMaxSize: %d ", bs)) - } - if l := h.CompressionLevel; l != 0 { - s.WriteString(fmt.Sprintf("CompressionLevel: %d ", l)) - } - s.WriteByte('}') - - return s.String() -} diff --git a/vendor/github.com/pierrec/lz4/reader.go b/vendor/github.com/pierrec/lz4/reader.go deleted file mode 100644 index f08db47d..00000000 --- a/vendor/github.com/pierrec/lz4/reader.go +++ /dev/null @@ -1,295 +0,0 @@ -package lz4 - -import ( - "encoding/binary" - "fmt" - "io" - "io/ioutil" - - "github.com/pierrec/lz4/internal/xxh32" -) - -// Reader implements the LZ4 frame decoder. -// The Header is set after the first call to Read(). -// The Header may change between Read() calls in case of concatenated frames. -type Reader struct { - Header - - buf [8]byte // Scrap buffer. - pos int64 // Current position in src. - src io.Reader // Source. - zdata []byte // Compressed data. - data []byte // Uncompressed data. - idx int // Index of unread bytes into data. - checksum xxh32.XXHZero // Frame hash. -} - -// NewReader returns a new LZ4 frame decoder. -// No access to the underlying io.Reader is performed. -func NewReader(src io.Reader) *Reader { - r := &Reader{src: src} - return r -} - -// readHeader checks the frame magic number and parses the frame descriptoz. -// Skippable frames are supported even as a first frame although the LZ4 -// specifications recommends skippable frames not to be used as first frames. -func (z *Reader) readHeader(first bool) error { - defer z.checksum.Reset() - - buf := z.buf[:] - for { - magic, err := z.readUint32() - if err != nil { - z.pos += 4 - if !first && err == io.ErrUnexpectedEOF { - return io.EOF - } - return err - } - if magic == frameMagic { - break - } - if magic>>8 != frameSkipMagic>>8 { - return ErrInvalid - } - skipSize, err := z.readUint32() - if err != nil { - return err - } - z.pos += 4 - m, err := io.CopyN(ioutil.Discard, z.src, int64(skipSize)) - if err != nil { - return err - } - z.pos += m - } - - // Header. - if _, err := io.ReadFull(z.src, buf[:2]); err != nil { - return err - } - z.pos += 8 - - b := buf[0] - if v := b >> 6; v != Version { - return fmt.Errorf("lz4: invalid version: got %d; expected %d", v, Version) - } - if b>>5&1 == 0 { - return fmt.Errorf("lz4: block dependency not supported") - } - z.BlockChecksum = b>>4&1 > 0 - frameSize := b>>3&1 > 0 - z.NoChecksum = b>>2&1 == 0 - - bmsID := buf[1] >> 4 & 0x7 - bSize, ok := bsMapID[bmsID] - if !ok { - return fmt.Errorf("lz4: invalid block max size ID: %d", bmsID) - } - z.BlockMaxSize = bSize - - // Allocate the compressed/uncompressed buffers. - // The compressed buffer cannot exceed the uncompressed one. - if n := 2 * bSize; cap(z.zdata) < n { - z.zdata = make([]byte, n, n) - } - if debugFlag { - debug("header block max size id=%d size=%d", bmsID, bSize) - } - z.zdata = z.zdata[:bSize] - z.data = z.zdata[:cap(z.zdata)][bSize:] - z.idx = len(z.data) - - z.checksum.Write(buf[0:2]) - - if frameSize { - buf := buf[:8] - if _, err := io.ReadFull(z.src, buf); err != nil { - return err - } - z.Size = binary.LittleEndian.Uint64(buf) - z.pos += 8 - z.checksum.Write(buf) - } - - // Header checksum. - if _, err := io.ReadFull(z.src, buf[:1]); err != nil { - return err - } - z.pos++ - if h := byte(z.checksum.Sum32() >> 8 & 0xFF); h != buf[0] { - return fmt.Errorf("lz4: invalid header checksum: got %x; expected %x", buf[0], h) - } - - z.Header.done = true - if debugFlag { - debug("header read: %v", z.Header) - } - - return nil -} - -// Read decompresses data from the underlying source into the supplied buffer. -// -// Since there can be multiple streams concatenated, Header values may -// change between calls to Read(). If that is the case, no data is actually read from -// the underlying io.Reader, to allow for potential input buffer resizing. -func (z *Reader) Read(buf []byte) (int, error) { - if debugFlag { - debug("Read buf len=%d", len(buf)) - } - if !z.Header.done { - if err := z.readHeader(true); err != nil { - return 0, err - } - if debugFlag { - debug("header read OK compressed buffer %d / %d uncompressed buffer %d : %d index=%d", - len(z.zdata), cap(z.zdata), len(z.data), cap(z.data), z.idx) - } - } - - if len(buf) == 0 { - return 0, nil - } - - if z.idx == len(z.data) { - // No data ready for reading, process the next block. - if debugFlag { - debug("reading block from writer") - } - // Block length: 0 = end of frame, highest bit set: uncompressed. - bLen, err := z.readUint32() - if err != nil { - return 0, err - } - z.pos += 4 - - if bLen == 0 { - // End of frame reached. - if !z.NoChecksum { - // Validate the frame checksum. - checksum, err := z.readUint32() - if err != nil { - return 0, err - } - if debugFlag { - debug("frame checksum got=%x / want=%x", z.checksum.Sum32(), checksum) - } - z.pos += 4 - if h := z.checksum.Sum32(); checksum != h { - return 0, fmt.Errorf("lz4: invalid frame checksum: got %x; expected %x", h, checksum) - } - } - - // Get ready for the next concatenated frame and keep the position. - pos := z.pos - z.Reset(z.src) - z.pos = pos - - // Since multiple frames can be concatenated, check for more. - return 0, z.readHeader(false) - } - - if debugFlag { - debug("raw block size %d", bLen) - } - if bLen&compressedBlockFlag > 0 { - // Uncompressed block. - bLen &= compressedBlockMask - if debugFlag { - debug("uncompressed block size %d", bLen) - } - if int(bLen) > cap(z.data) { - return 0, fmt.Errorf("lz4: invalid block size: %d", bLen) - } - z.data = z.data[:bLen] - if _, err := io.ReadFull(z.src, z.data); err != nil { - return 0, err - } - z.pos += int64(bLen) - - if z.BlockChecksum { - checksum, err := z.readUint32() - if err != nil { - return 0, err - } - z.pos += 4 - - if h := xxh32.ChecksumZero(z.data); h != checksum { - return 0, fmt.Errorf("lz4: invalid block checksum: got %x; expected %x", h, checksum) - } - } - - } else { - // Compressed block. - if debugFlag { - debug("compressed block size %d", bLen) - } - if int(bLen) > cap(z.data) { - return 0, fmt.Errorf("lz4: invalid block size: %d", bLen) - } - zdata := z.zdata[:bLen] - if _, err := io.ReadFull(z.src, zdata); err != nil { - return 0, err - } - z.pos += int64(bLen) - - if z.BlockChecksum { - checksum, err := z.readUint32() - if err != nil { - return 0, err - } - z.pos += 4 - - if h := xxh32.ChecksumZero(zdata); h != checksum { - return 0, fmt.Errorf("lz4: invalid block checksum: got %x; expected %x", h, checksum) - } - } - - n, err := UncompressBlock(zdata, z.data) - if err != nil { - return 0, err - } - z.data = z.data[:n] - } - - if !z.NoChecksum { - z.checksum.Write(z.data) - if debugFlag { - debug("current frame checksum %x", z.checksum.Sum32()) - } - } - z.idx = 0 - } - - n := copy(buf, z.data[z.idx:]) - z.idx += n - if debugFlag { - debug("copied %d bytes to input", n) - } - - return n, nil -} - -// Reset discards the Reader's state and makes it equivalent to the -// result of its original state from NewReader, but reading from r instead. -// This permits reusing a Reader rather than allocating a new one. -func (z *Reader) Reset(r io.Reader) { - z.Header = Header{} - z.pos = 0 - z.src = r - z.zdata = z.zdata[:0] - z.data = z.data[:0] - z.idx = 0 - z.checksum.Reset() -} - -// readUint32 reads an uint32 into the supplied buffer. -// The idea is to make use of the already allocated buffers avoiding additional allocations. -func (z *Reader) readUint32() (uint32, error) { - buf := z.buf[:4] - _, err := io.ReadFull(z.src, buf) - x := binary.LittleEndian.Uint32(buf) - return x, err -} diff --git a/vendor/github.com/pierrec/lz4/.gitignore b/vendor/github.com/pierrec/lz4/v4/.gitignore similarity index 94% rename from vendor/github.com/pierrec/lz4/.gitignore rename to vendor/github.com/pierrec/lz4/v4/.gitignore index e48bab32..5d7e88de 100644 --- a/vendor/github.com/pierrec/lz4/.gitignore +++ b/vendor/github.com/pierrec/lz4/v4/.gitignore @@ -30,4 +30,7 @@ Temporary Items # End of https://www.gitignore.io/api/macos -lz4c/lz4c +cmd/*/*exe +.idea + +fuzz/*.zip diff --git a/vendor/github.com/pierrec/lz4/LICENSE b/vendor/github.com/pierrec/lz4/v4/LICENSE similarity index 100% rename from vendor/github.com/pierrec/lz4/LICENSE rename to vendor/github.com/pierrec/lz4/v4/LICENSE diff --git a/vendor/github.com/pierrec/lz4/v4/README.md b/vendor/github.com/pierrec/lz4/v4/README.md new file mode 100644 index 00000000..df027e2c --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/README.md @@ -0,0 +1,90 @@ +# lz4 : LZ4 compression in pure Go + +[![Go Reference](https://pkg.go.dev/badge/github.com/pierrec/lz4/v4.svg)](https://pkg.go.dev/github.com/pierrec/lz4/v4) +[![CI](https://github.com/pierrec/lz4/workflows/ci/badge.svg)](https://github.com/pierrec/lz4/actions) +[![Go Report Card](https://goreportcard.com/badge/github.com/pierrec/lz4)](https://goreportcard.com/report/github.com/pierrec/lz4) +[![GitHub tag (latest SemVer)](https://img.shields.io/github/tag/pierrec/lz4.svg?style=social)](https://github.com/pierrec/lz4/tags) + +## Overview + +This package provides a streaming interface to [LZ4 data streams](http://fastcompression.blogspot.fr/2013/04/lz4-streaming-format-final.html) as well as low level compress and uncompress functions for LZ4 data blocks. +The implementation is based on the reference C [one](https://github.com/lz4/lz4). + +## Install + +Assuming you have the go toolchain installed: + +``` +go get github.com/pierrec/lz4/v4 +``` + +There is a command line interface tool to compress and decompress LZ4 files. + +``` +go install github.com/pierrec/lz4/v4/cmd/lz4c +``` + +Usage + +``` +Usage of lz4c: + -version + print the program version + +Subcommands: +Compress the given files or from stdin to stdout. +compress [arguments] [ ...] + -bc + enable block checksum + -l int + compression level (0=fastest) + -sc + disable stream checksum + -size string + block max size [64K,256K,1M,4M] (default "4M") + +Uncompress the given files or from stdin to stdout. +uncompress [arguments] [ ...] + +``` + + +## Example + +``` +// Compress and uncompress an input string. +s := "hello world" +r := strings.NewReader(s) + +// The pipe will uncompress the data from the writer. +pr, pw := io.Pipe() +zw := lz4.NewWriter(pw) +zr := lz4.NewReader(pr) + +go func() { + // Compress the input string. + _, _ = io.Copy(zw, r) + _ = zw.Close() // Make sure the writer is closed + _ = pw.Close() // Terminate the pipe +}() + +_, _ = io.Copy(os.Stdout, zr) + +// Output: +// hello world +``` + +## Contributing + +Contributions are very welcome for bug fixing, performance improvements...! + +- Open an issue with a proper description +- Send a pull request with appropriate test case(s) + +## Contributors + +Thanks to all [contributors](https://github.com/pierrec/lz4/graphs/contributors) so far! + +Special thanks to [@Zariel](https://github.com/Zariel) for his asm implementation of the decoder. + +Special thanks to [@klauspost](https://github.com/klauspost) for his work on optimizing the code. diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go new file mode 100644 index 00000000..f3826494 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go @@ -0,0 +1,469 @@ +package lz4block + +import ( + "encoding/binary" + "math/bits" + "sync" + + "github.com/pierrec/lz4/v4/internal/lz4errors" +) + +const ( + // The following constants are used to setup the compression algorithm. + minMatch = 4 // the minimum size of the match sequence size (4 bytes) + winSizeLog = 16 // LZ4 64Kb window size limit + winSize = 1 << winSizeLog + winMask = winSize - 1 // 64Kb window of previous data for dependent blocks + + // hashLog determines the size of the hash table used to quickly find a previous match position. + // Its value influences the compression speed and memory usage, the lower the faster, + // but at the expense of the compression ratio. + // 16 seems to be the best compromise for fast compression. + hashLog = 16 + htSize = 1 << hashLog + + mfLimit = 10 + minMatch // The last match cannot start within the last 14 bytes. +) + +func recoverBlock(e *error) { + if r := recover(); r != nil && *e == nil { + *e = lz4errors.ErrInvalidSourceShortBuffer + } +} + +// blockHash hashes the lower 6 bytes into a value < htSize. +func blockHash(x uint64) uint32 { + const prime6bytes = 227718039650203 + return uint32(((x << (64 - 48)) * prime6bytes) >> (64 - hashLog)) +} + +func CompressBlockBound(n int) int { + return n + n/255 + 16 +} + +func UncompressBlock(src, dst []byte) (int, error) { + if len(src) == 0 { + return 0, nil + } + if di := decodeBlock(dst, src); di >= 0 { + return di, nil + } + return 0, lz4errors.ErrInvalidSourceShortBuffer +} + +type Compressor struct { + // Offsets are at most 64kiB, so we can store only the lower 16 bits of + // match positions: effectively, an offset from some 64kiB block boundary. + // + // When we retrieve such an offset, we interpret it as relative to the last + // block boundary si &^ 0xffff, or the one before, (si &^ 0xffff) - 0x10000, + // depending on which of these is inside the current window. If a table + // entry was generated more than 64kiB back in the input, we find out by + // inspecting the input stream. + table [htSize]uint16 + + needsReset bool +} + +// Get returns the position of a presumptive match for the hash h. +// The match may be a false positive due to a hash collision or an old entry. +// If si < winSize, the return value may be negative. +func (c *Compressor) get(h uint32, si int) int { + h &= htSize - 1 + i := int(c.table[h]) + i += si &^ winMask + if i >= si { + // Try previous 64kiB block (negative when in first block). + i -= winSize + } + return i +} + +func (c *Compressor) put(h uint32, si int) { + h &= htSize - 1 + c.table[h] = uint16(si) +} + +var compressorPool = sync.Pool{New: func() interface{} { return new(Compressor) }} + +func CompressBlock(src, dst []byte) (int, error) { + c := compressorPool.Get().(*Compressor) + n, err := c.CompressBlock(src, dst) + compressorPool.Put(c) + return n, err +} + +func (c *Compressor) CompressBlock(src, dst []byte) (int, error) { + if c.needsReset { + // Zero out reused table to avoid non-deterministic output (issue #65). + c.table = [htSize]uint16{} + } + c.needsReset = true // Only false on first call. + + // Return 0, nil only if the destination buffer size is < CompressBlockBound. + isNotCompressible := len(dst) < CompressBlockBound(len(src)) + + // adaptSkipLog sets how quickly the compressor begins skipping blocks when data is incompressible. + // This significantly speeds up incompressible data and usually has very small impact on compression. + // bytes to skip = 1 + (bytes since last match >> adaptSkipLog) + const adaptSkipLog = 7 + + // si: Current position of the search. + // anchor: Position of the current literals. + var si, di, anchor int + sn := len(src) - mfLimit + if sn <= 0 { + goto lastLiterals + } + + // Fast scan strategy: the hash table only stores the last 4 bytes sequences. + for si < sn { + // Hash the next 6 bytes (sequence)... + match := binary.LittleEndian.Uint64(src[si:]) + h := blockHash(match) + h2 := blockHash(match >> 8) + + // We check a match at s, s+1 and s+2 and pick the first one we get. + // Checking 3 only requires us to load the source one. + ref := c.get(h, si) + ref2 := c.get(h2, si) + c.put(h, si) + c.put(h2, si+1) + + offset := si - ref + + if offset <= 0 || offset >= winSize || uint32(match) != binary.LittleEndian.Uint32(src[ref:]) { + // No match. Start calculating another hash. + // The processor can usually do this out-of-order. + h = blockHash(match >> 16) + ref3 := c.get(h, si+2) + + // Check the second match at si+1 + si += 1 + offset = si - ref2 + + if offset <= 0 || offset >= winSize || uint32(match>>8) != binary.LittleEndian.Uint32(src[ref2:]) { + // No match. Check the third match at si+2 + si += 1 + offset = si - ref3 + c.put(h, si) + + if offset <= 0 || offset >= winSize || uint32(match>>16) != binary.LittleEndian.Uint32(src[ref3:]) { + // Skip one extra byte (at si+3) before we check 3 matches again. + si += 2 + (si-anchor)>>adaptSkipLog + continue + } + } + } + + // Match found. + lLen := si - anchor // Literal length. + // We already matched 4 bytes. + mLen := 4 + + // Extend backwards if we can, reducing literals. + tOff := si - offset - 1 + for lLen > 0 && tOff >= 0 && src[si-1] == src[tOff] { + si-- + tOff-- + lLen-- + mLen++ + } + + // Add the match length, so we continue search at the end. + // Use mLen to store the offset base. + si, mLen = si+mLen, si+minMatch + + // Find the longest match by looking by batches of 8 bytes. + for si+8 < sn { + x := binary.LittleEndian.Uint64(src[si:]) ^ binary.LittleEndian.Uint64(src[si-offset:]) + if x == 0 { + si += 8 + } else { + // Stop is first non-zero byte. + si += bits.TrailingZeros64(x) >> 3 + break + } + } + + mLen = si - mLen + if mLen < 0xF { + dst[di] = byte(mLen) + } else { + dst[di] = 0xF + } + + // Encode literals length. + if lLen < 0xF { + dst[di] |= byte(lLen << 4) + } else { + dst[di] |= 0xF0 + di++ + l := lLen - 0xF + for ; l >= 0xFF; l -= 0xFF { + dst[di] = 0xFF + di++ + } + dst[di] = byte(l) + } + di++ + + // Literals. + if di+lLen > len(dst) { + return 0, lz4errors.ErrInvalidSourceShortBuffer + } + copy(dst[di:di+lLen], src[anchor:anchor+lLen]) + di += lLen + 2 + anchor = si + + // Encode offset. + if di > len(dst) { + return 0, lz4errors.ErrInvalidSourceShortBuffer + } + dst[di-2], dst[di-1] = byte(offset), byte(offset>>8) + + // Encode match length part 2. + if mLen >= 0xF { + for mLen -= 0xF; mLen >= 0xFF && di < len(dst); mLen -= 0xFF { + dst[di] = 0xFF + di++ + } + if di >= len(dst) { + return 0, lz4errors.ErrInvalidSourceShortBuffer + } + dst[di] = byte(mLen) + di++ + } + // Check if we can load next values. + if si >= sn { + break + } + // Hash match end-2 + h = blockHash(binary.LittleEndian.Uint64(src[si-2:])) + c.put(h, si-2) + } + +lastLiterals: + if isNotCompressible && anchor == 0 { + // Incompressible. + return 0, nil + } + + // Last literals. + if di >= len(dst) { + return 0, lz4errors.ErrInvalidSourceShortBuffer + } + lLen := len(src) - anchor + if lLen < 0xF { + dst[di] = byte(lLen << 4) + } else { + dst[di] = 0xF0 + di++ + for lLen -= 0xF; lLen >= 0xFF && di < len(dst); lLen -= 0xFF { + dst[di] = 0xFF + di++ + } + if di >= len(dst) { + return 0, lz4errors.ErrInvalidSourceShortBuffer + } + dst[di] = byte(lLen) + } + di++ + + // Write the last literals. + if isNotCompressible && di >= anchor { + // Incompressible. + return 0, nil + } + if di+len(src)-anchor > len(dst) { + return 0, lz4errors.ErrInvalidSourceShortBuffer + } + di += copy(dst[di:di+len(src)-anchor], src[anchor:]) + return di, nil +} + +// blockHash hashes 4 bytes into a value < winSize. +func blockHashHC(x uint32) uint32 { + const hasher uint32 = 2654435761 // Knuth multiplicative hash. + return x * hasher >> (32 - winSizeLog) +} + +type CompressorHC struct { + // hashTable: stores the last position found for a given hash + // chainTable: stores previous positions for a given hash + hashTable, chainTable [htSize]int + needsReset bool +} + +var compressorHCPool = sync.Pool{New: func() interface{} { return new(CompressorHC) }} + +func CompressBlockHC(src, dst []byte, depth CompressionLevel) (int, error) { + c := compressorHCPool.Get().(*CompressorHC) + n, err := c.CompressBlock(src, dst, depth) + compressorHCPool.Put(c) + return n, err +} + +func (c *CompressorHC) CompressBlock(src, dst []byte, depth CompressionLevel) (_ int, err error) { + if c.needsReset { + // Zero out reused table to avoid non-deterministic output (issue #65). + c.hashTable = [htSize]int{} + c.chainTable = [htSize]int{} + } + c.needsReset = true // Only false on first call. + + defer recoverBlock(&err) + + // Return 0, nil only if the destination buffer size is < CompressBlockBound. + isNotCompressible := len(dst) < CompressBlockBound(len(src)) + + // adaptSkipLog sets how quickly the compressor begins skipping blocks when data is incompressible. + // This significantly speeds up incompressible data and usually has very small impact on compression. + // bytes to skip = 1 + (bytes since last match >> adaptSkipLog) + const adaptSkipLog = 7 + + var si, di, anchor int + sn := len(src) - mfLimit + if sn <= 0 { + goto lastLiterals + } + + if depth == 0 { + depth = winSize + } + + for si < sn { + // Hash the next 4 bytes (sequence). + match := binary.LittleEndian.Uint32(src[si:]) + h := blockHashHC(match) + + // Follow the chain until out of window and give the longest match. + mLen := 0 + offset := 0 + for next, try := c.hashTable[h], depth; try > 0 && next > 0 && si-next < winSize; next, try = c.chainTable[next&winMask], try-1 { + // The first (mLen==0) or next byte (mLen>=minMatch) at current match length + // must match to improve on the match length. + if src[next+mLen] != src[si+mLen] { + continue + } + ml := 0 + // Compare the current position with a previous with the same hash. + for ml < sn-si { + x := binary.LittleEndian.Uint64(src[next+ml:]) ^ binary.LittleEndian.Uint64(src[si+ml:]) + if x == 0 { + ml += 8 + } else { + // Stop is first non-zero byte. + ml += bits.TrailingZeros64(x) >> 3 + break + } + } + if ml < minMatch || ml <= mLen { + // Match too small (>adaptSkipLog + continue + } + + // Match found. + // Update hash/chain tables with overlapping bytes: + // si already hashed, add everything from si+1 up to the match length. + winStart := si + 1 + if ws := si + mLen - winSize; ws > winStart { + winStart = ws + } + for si, ml := winStart, si+mLen; si < ml; { + match >>= 8 + match |= uint32(src[si+3]) << 24 + h := blockHashHC(match) + c.chainTable[si&winMask] = c.hashTable[h] + c.hashTable[h] = si + si++ + } + + lLen := si - anchor + si += mLen + mLen -= minMatch // Match length does not include minMatch. + + if mLen < 0xF { + dst[di] = byte(mLen) + } else { + dst[di] = 0xF + } + + // Encode literals length. + if lLen < 0xF { + dst[di] |= byte(lLen << 4) + } else { + dst[di] |= 0xF0 + di++ + l := lLen - 0xF + for ; l >= 0xFF; l -= 0xFF { + dst[di] = 0xFF + di++ + } + dst[di] = byte(l) + } + di++ + + // Literals. + copy(dst[di:di+lLen], src[anchor:anchor+lLen]) + di += lLen + anchor = si + + // Encode offset. + di += 2 + dst[di-2], dst[di-1] = byte(offset), byte(offset>>8) + + // Encode match length part 2. + if mLen >= 0xF { + for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF { + dst[di] = 0xFF + di++ + } + dst[di] = byte(mLen) + di++ + } + } + + if isNotCompressible && anchor == 0 { + // Incompressible. + return 0, nil + } + + // Last literals. +lastLiterals: + lLen := len(src) - anchor + if lLen < 0xF { + dst[di] = byte(lLen << 4) + } else { + dst[di] = 0xF0 + di++ + lLen -= 0xF + for ; lLen >= 0xFF; lLen -= 0xFF { + dst[di] = 0xFF + di++ + } + dst[di] = byte(lLen) + } + di++ + + // Write the last literals. + if isNotCompressible && di >= anchor { + // Incompressible. + return 0, nil + } + di += copy(dst[di:di+len(src)-anchor], src[anchor:]) + return di, nil +} diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go new file mode 100644 index 00000000..e6cf88d7 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go @@ -0,0 +1,88 @@ +// Package lz4block provides LZ4 BlockSize types and pools of buffers. +package lz4block + +import "sync" + +const ( + Block64Kb uint32 = 1 << (16 + iota*2) + Block256Kb + Block1Mb + Block4Mb + Block8Mb = 2 * Block4Mb + legacyBlockSize = Block8Mb + Block8Mb/255 + 16 // CompressBound(Block8Mb) +) + +var ( + BlockPool64K = sync.Pool{New: func() interface{} { return make([]byte, Block64Kb) }} + BlockPool256K = sync.Pool{New: func() interface{} { return make([]byte, Block256Kb) }} + BlockPool1M = sync.Pool{New: func() interface{} { return make([]byte, Block1Mb) }} + BlockPool4M = sync.Pool{New: func() interface{} { return make([]byte, Block4Mb) }} + BlockPool8M = sync.Pool{New: func() interface{} { return make([]byte, legacyBlockSize) }} +) + +func Index(b uint32) BlockSizeIndex { + switch b { + case Block64Kb: + return 4 + case Block256Kb: + return 5 + case Block1Mb: + return 6 + case Block4Mb: + return 7 + case Block8Mb: // only valid in legacy mode + return 3 + } + return 0 +} + +func IsValid(b uint32) bool { + return Index(b) > 0 +} + +type BlockSizeIndex uint8 + +func (b BlockSizeIndex) IsValid() bool { + switch b { + case 4, 5, 6, 7: + return true + } + return false +} + +func (b BlockSizeIndex) Get() []byte { + var buf interface{} + switch b { + case 4: + buf = BlockPool64K.Get() + case 5: + buf = BlockPool256K.Get() + case 6: + buf = BlockPool1M.Get() + case 7: + buf = BlockPool4M.Get() + case 3: + buf = BlockPool8M.Get() + } + return buf.([]byte) +} + +func Put(buf []byte) { + // Safeguard: do not allow invalid buffers. + switch c := cap(buf); uint32(c) { + case Block64Kb: + BlockPool64K.Put(buf[:c]) + case Block256Kb: + BlockPool256K.Put(buf[:c]) + case Block1Mb: + BlockPool1M.Put(buf[:c]) + case Block4Mb: + BlockPool4M.Put(buf[:c]) + case legacyBlockSize: + BlockPool8M.Put(buf[:c]) + } +} + +type CompressionLevel uint32 + +const Fast CompressionLevel = 0 diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_amd64.s b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_amd64.s new file mode 100644 index 00000000..be79faa3 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_amd64.s @@ -0,0 +1,369 @@ +// +build !appengine +// +build gc +// +build !noasm + +#include "textflag.h" + +// AX scratch +// BX scratch +// CX scratch +// DX token +// +// DI &dst +// SI &src +// R8 &dst + len(dst) +// R9 &src + len(src) +// R11 &dst +// R12 short output end +// R13 short input end +// func decodeBlock(dst, src []byte) int +// using 50 bytes of stack currently +TEXT ·decodeBlock(SB), NOSPLIT, $64-56 + MOVQ dst_base+0(FP), DI + MOVQ DI, R11 + MOVQ dst_len+8(FP), R8 + ADDQ DI, R8 + + MOVQ src_base+24(FP), SI + MOVQ src_len+32(FP), R9 + CMPQ R9, $0 + JE err_corrupt + ADDQ SI, R9 + + // shortcut ends + // short output end + MOVQ R8, R12 + SUBQ $32, R12 + // short input end + MOVQ R9, R13 + SUBQ $16, R13 + +loop: + // for si < len(src) + CMPQ SI, R9 + JGE end + + // token := uint32(src[si]) + MOVBQZX (SI), DX + INCQ SI + + // lit_len = token >> 4 + // if lit_len > 0 + // CX = lit_len + MOVQ DX, CX + SHRQ $4, CX + + // if lit_len != 0xF + CMPQ CX, $0xF + JEQ lit_len_loop_pre + CMPQ DI, R12 + JGE lit_len_loop_pre + CMPQ SI, R13 + JGE lit_len_loop_pre + + // copy shortcut + + // A two-stage shortcut for the most common case: + // 1) If the literal length is 0..14, and there is enough space, + // enter the shortcut and copy 16 bytes on behalf of the literals + // (in the fast mode, only 8 bytes can be safely copied this way). + // 2) Further if the match length is 4..18, copy 18 bytes in a similar + // manner; but we ensure that there's enough space in the output for + // those 18 bytes earlier, upon entering the shortcut (in other words, + // there is a combined check for both stages). + + // copy literal + MOVOU (SI), X0 + MOVOU X0, (DI) + ADDQ CX, DI + ADDQ CX, SI + + MOVQ DX, CX + ANDQ $0xF, CX + + // The second stage: prepare for match copying, decode full info. + // If it doesn't work out, the info won't be wasted. + // offset := uint16(data[:2]) + MOVWQZX (SI), DX + ADDQ $2, SI + + MOVQ DI, AX + SUBQ DX, AX + CMPQ AX, DI + JGT err_short_buf + + // if we can't do the second stage then jump straight to read the + // match length, we already have the offset. + CMPQ CX, $0xF + JEQ match_len_loop_pre + CMPQ DX, $8 + JLT match_len_loop_pre + CMPQ AX, R11 + JLT err_short_buf + + // memcpy(op + 0, match + 0, 8); + MOVQ (AX), BX + MOVQ BX, (DI) + // memcpy(op + 8, match + 8, 8); + MOVQ 8(AX), BX + MOVQ BX, 8(DI) + // memcpy(op +16, match +16, 2); + MOVW 16(AX), BX + MOVW BX, 16(DI) + + LEAQ 4(DI)(CX*1), DI // minmatch + + // shortcut complete, load next token + JMP loop + +lit_len_loop_pre: + // if lit_len > 0 + CMPQ CX, $0 + JEQ offset + CMPQ CX, $0xF + JNE copy_literal + +lit_len_loop: + // for src[si] == 0xFF + CMPB (SI), $0xFF + JNE lit_len_finalise + + // bounds check src[si+1] + LEAQ 1(SI), AX + CMPQ AX, R9 + JGT err_short_buf + + // lit_len += 0xFF + ADDQ $0xFF, CX + INCQ SI + JMP lit_len_loop + +lit_len_finalise: + // lit_len += int(src[si]) + // si++ + MOVBQZX (SI), AX + ADDQ AX, CX + INCQ SI + +copy_literal: + // bounds check src and dst + LEAQ (SI)(CX*1), AX + CMPQ AX, R9 + JGT err_short_buf + + LEAQ (DI)(CX*1), AX + CMPQ AX, R8 + JGT err_short_buf + + // whats a good cut off to call memmove? + CMPQ CX, $16 + JGT memmove_lit + + // if len(dst[di:]) < 16 + MOVQ R8, AX + SUBQ DI, AX + CMPQ AX, $16 + JLT memmove_lit + + // if len(src[si:]) < 16 + MOVQ R9, AX + SUBQ SI, AX + CMPQ AX, $16 + JLT memmove_lit + + MOVOU (SI), X0 + MOVOU X0, (DI) + + JMP finish_lit_copy + +memmove_lit: + // memmove(to, from, len) + MOVQ DI, 0(SP) + MOVQ SI, 8(SP) + MOVQ CX, 16(SP) + // spill + MOVQ DI, 24(SP) + MOVQ SI, 32(SP) + MOVQ CX, 40(SP) // need len to inc SI, DI after + MOVB DX, 48(SP) + CALL runtime·memmove(SB) + + // restore registers + MOVQ 24(SP), DI + MOVQ 32(SP), SI + MOVQ 40(SP), CX + MOVB 48(SP), DX + + // recalc initial values + MOVQ dst_base+0(FP), R8 + MOVQ R8, R11 + ADDQ dst_len+8(FP), R8 + MOVQ src_base+24(FP), R9 + ADDQ src_len+32(FP), R9 + MOVQ R8, R12 + SUBQ $32, R12 + MOVQ R9, R13 + SUBQ $16, R13 + +finish_lit_copy: + ADDQ CX, SI + ADDQ CX, DI + + CMPQ SI, R9 + JGE end + +offset: + // CX := mLen + // free up DX to use for offset + MOVQ DX, CX + + LEAQ 2(SI), AX + CMPQ AX, R9 + JGT err_short_buf + + // offset + // DX := int(src[si]) | int(src[si+1])<<8 + MOVWQZX (SI), DX + ADDQ $2, SI + + // 0 offset is invalid + CMPQ DX, $0 + JEQ err_corrupt + + ANDB $0xF, CX + +match_len_loop_pre: + // if mlen != 0xF + CMPB CX, $0xF + JNE copy_match + +match_len_loop: + // for src[si] == 0xFF + // lit_len += 0xFF + CMPB (SI), $0xFF + JNE match_len_finalise + + // bounds check src[si+1] + LEAQ 1(SI), AX + CMPQ AX, R9 + JGT err_short_buf + + ADDQ $0xFF, CX + INCQ SI + JMP match_len_loop + +match_len_finalise: + // lit_len += int(src[si]) + // si++ + MOVBQZX (SI), AX + ADDQ AX, CX + INCQ SI + +copy_match: + // mLen += minMatch + ADDQ $4, CX + + // check we have match_len bytes left in dst + // di+match_len < len(dst) + LEAQ (DI)(CX*1), AX + CMPQ AX, R8 + JGT err_short_buf + + // DX = offset + // CX = match_len + // BX = &dst + (di - offset) + MOVQ DI, BX + SUBQ DX, BX + + // check BX is within dst + // if BX < &dst + CMPQ BX, R11 + JLT err_short_buf + + // if offset + match_len < di + LEAQ (BX)(CX*1), AX + CMPQ DI, AX + JGT copy_interior_match + + // AX := len(dst[:di]) + // MOVQ DI, AX + // SUBQ R11, AX + + // copy 16 bytes at a time + // if di-offset < 16 copy 16-(di-offset) bytes to di + // then do the remaining + +copy_match_loop: + // for match_len >= 0 + // dst[di] = dst[i] + // di++ + // i++ + MOVB (BX), AX + MOVB AX, (DI) + INCQ DI + INCQ BX + DECQ CX + + CMPQ CX, $0 + JGT copy_match_loop + + JMP loop + +copy_interior_match: + CMPQ CX, $16 + JGT memmove_match + + // if len(dst[di:]) < 16 + MOVQ R8, AX + SUBQ DI, AX + CMPQ AX, $16 + JLT memmove_match + + MOVOU (BX), X0 + MOVOU X0, (DI) + + ADDQ CX, DI + JMP loop + +memmove_match: + // memmove(to, from, len) + MOVQ DI, 0(SP) + MOVQ BX, 8(SP) + MOVQ CX, 16(SP) + // spill + MOVQ DI, 24(SP) + MOVQ SI, 32(SP) + MOVQ CX, 40(SP) // need len to inc SI, DI after + CALL runtime·memmove(SB) + + // restore registers + MOVQ 24(SP), DI + MOVQ 32(SP), SI + MOVQ 40(SP), CX + + // recalc initial values + MOVQ dst_base+0(FP), R8 + MOVQ R8, R11 // TODO: make these sensible numbers + ADDQ dst_len+8(FP), R8 + MOVQ src_base+24(FP), R9 + ADDQ src_len+32(FP), R9 + MOVQ R8, R12 + SUBQ $32, R12 + MOVQ R9, R13 + SUBQ $16, R13 + + ADDQ CX, DI + JMP loop + +err_corrupt: + MOVQ $-1, ret+48(FP) + RET + +err_short_buf: + MOVQ $-2, ret+48(FP) + RET + +end: + SUBQ R11, DI + MOVQ DI, ret+48(FP) + RET diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s new file mode 100644 index 00000000..64be9adc --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s @@ -0,0 +1,197 @@ +// +build gc +// +build !noasm + +#include "textflag.h" + +// Register allocation. +#define dst R0 +#define dstorig R1 +#define src R2 +#define dstend R3 +#define srcend R4 +#define match R5 // Match address. +#define token R6 +#define len R7 // Literal and match lengths. +#define offset R6 // Match offset; overlaps with token. +#define tmp1 R8 +#define tmp2 R9 +#define tmp3 R12 + +#define minMatch $4 + +// func decodeBlock(dst, src []byte) int +TEXT ·decodeBlock(SB), NOFRAME|NOSPLIT, $-4-28 + MOVW dst_base +0(FP), dst + MOVW dst_len +4(FP), dstend + MOVW src_base+12(FP), src + MOVW src_len +16(FP), srcend + + CMP $0, srcend + BEQ shortSrc + + ADD dst, dstend + ADD src, srcend + + MOVW dst, dstorig + +loop: + // Read token. Extract literal length. + MOVBU.P 1(src), token + MOVW token >> 4, len + CMP $15, len + BNE readLitlenDone + +readLitlenLoop: + CMP src, srcend + BEQ shortSrc + MOVBU.P 1(src), tmp1 + ADD tmp1, len + CMP $255, tmp1 + BEQ readLitlenLoop + +readLitlenDone: + CMP $0, len + BEQ copyLiteralDone + + // Bounds check dst+len and src+len. + ADD dst, len, tmp1 + CMP dstend, tmp1 + //BHI shortDst // Uncomment for distinct error codes. + ADD src, len, tmp2 + CMP.LS srcend, tmp2 + BHI shortSrc + + // Copy literal. + CMP $4, len + BLO copyLiteralFinish + + // Copy 0-3 bytes until src is aligned. + TST $1, src + MOVBU.NE.P 1(src), tmp1 + MOVB.NE.P tmp1, 1(dst) + SUB.NE $1, len + + TST $2, src + MOVHU.NE.P 2(src), tmp2 + MOVB.NE.P tmp2, 1(dst) + MOVW.NE tmp2 >> 8, tmp1 + MOVB.NE.P tmp1, 1(dst) + SUB.NE $2, len + + B copyLiteralLoopCond + +copyLiteralLoop: + // Aligned load, unaligned write. + MOVW.P 4(src), tmp1 + MOVW tmp1 >> 8, tmp2 + MOVB tmp2, 1(dst) + MOVW tmp1 >> 16, tmp3 + MOVB tmp3, 2(dst) + MOVW tmp1 >> 24, tmp2 + MOVB tmp2, 3(dst) + MOVB.P tmp1, 4(dst) +copyLiteralLoopCond: + // Loop until len-4 < 0. + SUB.S $4, len + BPL copyLiteralLoop + + // Restore len, which is now negative. + ADD $4, len + +copyLiteralFinish: + // Copy remaining 0-3 bytes. + TST $2, len + MOVHU.NE.P 2(src), tmp2 + MOVB.NE.P tmp2, 1(dst) + MOVW.NE tmp2 >> 8, tmp1 + MOVB.NE.P tmp1, 1(dst) + TST $1, len + MOVBU.NE.P 1(src), tmp1 + MOVB.NE.P tmp1, 1(dst) + +copyLiteralDone: + CMP src, srcend + BEQ end + + // Initial part of match length. + // This frees up the token register for reuse as offset. + AND $15, token, len + + // Read offset. + ADD $2, src + CMP srcend, src + BHI shortSrc + MOVBU -2(src), offset + MOVBU -1(src), tmp1 + ORR tmp1 << 8, offset + CMP $0, offset + BEQ corrupt + + // Read rest of match length. + CMP $15, len + BNE readMatchlenDone + +readMatchlenLoop: + CMP src, srcend + BEQ shortSrc + MOVBU.P 1(src), tmp1 + ADD tmp1, len + CMP $255, tmp1 + BEQ readMatchlenLoop + +readMatchlenDone: + // Bounds check dst+len+minMatch and match = dst-offset. + ADD dst, len, tmp1 + ADD minMatch, tmp1 + CMP dstend, tmp1 + //BHI shortDst // Uncomment for distinct error codes. + SUB offset, dst, match + CMP.LS match, dstorig + BHI corrupt + + // Since len+minMatch is at least four, we can do a 4× unrolled + // byte copy loop. Using MOVW instead of four byte loads is faster, + // but to remain portable we'd have to align match first, which is + // too expensive. By alternating loads and stores, we also handle + // the case offset < 4. +copyMatch4: + SUB.S $4, len + MOVBU.P 4(match), tmp1 + MOVB.P tmp1, 4(dst) + MOVBU -3(match), tmp2 + MOVB tmp2, -3(dst) + MOVBU -2(match), tmp3 + MOVB tmp3, -2(dst) + MOVBU -1(match), tmp1 + MOVB tmp1, -1(dst) + BPL copyMatch4 + + // Restore len, which is now negative. + ADD.S $4, len + BEQ copyMatchDone + +copyMatch: + // Finish with a byte-at-a-time copy. + SUB.S $1, len + MOVBU.P 1(match), tmp2 + MOVB.P tmp2, 1(dst) + BNE copyMatch + +copyMatchDone: + CMP src, srcend + BNE loop + +end: + SUB dstorig, dst, tmp1 + MOVW tmp1, ret+24(FP) + RET + + // The three error cases have distinct labels so we can put different + // return codes here when debugging, or if the error returns need to + // be changed. +shortDst: +shortSrc: +corrupt: + MOVW $-1, tmp1 + MOVW tmp1, ret+24(FP) + RET diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_asm.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_asm.go new file mode 100644 index 00000000..e26f8cd6 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_asm.go @@ -0,0 +1,9 @@ +// +build amd64 arm +// +build !appengine +// +build gc +// +build !noasm + +package lz4block + +//go:noescape +func decodeBlock(dst, src []byte) int diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go new file mode 100644 index 00000000..52df2f2b --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go @@ -0,0 +1,108 @@ +// +build !amd64,!arm appengine !gc noasm + +package lz4block + +import "encoding/binary" + +func decodeBlock(dst, src []byte) (ret int) { + // Restrict capacities so we don't read or write out of bounds. + dst = dst[:len(dst):len(dst)] + src = src[:len(src):len(src)] + + const hasError = -2 + defer func() { + if recover() != nil { + ret = hasError + } + }() + + var si, di uint + for { + // Literals and match lengths (token). + b := uint(src[si]) + si++ + + // Literals. + if lLen := b >> 4; lLen > 0 { + switch { + case lLen < 0xF && si+16 < uint(len(src)): + // Shortcut 1 + // if we have enough room in src and dst, and the literals length + // is small enough (0..14) then copy all 16 bytes, even if not all + // are part of the literals. + copy(dst[di:], src[si:si+16]) + si += lLen + di += lLen + if mLen := b & 0xF; mLen < 0xF { + // Shortcut 2 + // if the match length (4..18) fits within the literals, then copy + // all 18 bytes, even if not all are part of the literals. + mLen += 4 + if offset := u16(src[si:]); mLen <= offset { + i := di - offset + end := i + 18 + if end > uint(len(dst)) { + // The remaining buffer may not hold 18 bytes. + // See https://github.com/pierrec/lz4/issues/51. + end = uint(len(dst)) + } + copy(dst[di:], dst[i:end]) + si += 2 + di += mLen + continue + } + } + case lLen == 0xF: + for src[si] == 0xFF { + lLen += 0xFF + si++ + } + lLen += uint(src[si]) + si++ + fallthrough + default: + copy(dst[di:di+lLen], src[si:si+lLen]) + si += lLen + di += lLen + } + } + if si == uint(len(src)) { + return int(di) + } else if si > uint(len(src)) { + return hasError + } + + offset := u16(src[si:]) + if offset == 0 { + return hasError + } + si += 2 + + // Match. + mLen := b & 0xF + if mLen == 0xF { + for src[si] == 0xFF { + mLen += 0xFF + si++ + } + mLen += uint(src[si]) + si++ + } + mLen += minMatch + + // Copy the match. + expanded := dst[di-offset:] + if mLen > offset { + // Efficiently copy the match dst[di-offset:di] into the dst slice. + bytesToCopy := offset * (mLen / offset) + for n := offset; n <= bytesToCopy+offset; n *= 2 { + copy(expanded[n:], expanded[:n]) + } + di += bytesToCopy + mLen -= bytesToCopy + } + di += uint(copy(dst[di:di+mLen], expanded[:mLen])) + } +} + +func u16(p []byte) uint { return uint(binary.LittleEndian.Uint16(p)) } diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4errors/errors.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4errors/errors.go new file mode 100644 index 00000000..710ea428 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4errors/errors.go @@ -0,0 +1,19 @@ +package lz4errors + +type Error string + +func (e Error) Error() string { return string(e) } + +const ( + ErrInvalidSourceShortBuffer Error = "lz4: invalid source or destination buffer too short" + ErrInvalidFrame Error = "lz4: bad magic number" + ErrInternalUnhandledState Error = "lz4: unhandled state" + ErrInvalidHeaderChecksum Error = "lz4: invalid header checksum" + ErrInvalidBlockChecksum Error = "lz4: invalid block checksum" + ErrInvalidFrameChecksum Error = "lz4: invalid frame checksum" + ErrOptionInvalidCompressionLevel Error = "lz4: invalid compression level" + ErrOptionClosedOrError Error = "lz4: cannot apply options on closed or in error object" + ErrOptionInvalidBlockSize Error = "lz4: invalid block size" + ErrOptionNotApplicable Error = "lz4: option not applicable" + ErrWriterNotClosed Error = "lz4: writer not closed" +) diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go new file mode 100644 index 00000000..5e0c062e --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go @@ -0,0 +1,332 @@ +package lz4stream + +import ( + "encoding/binary" + "fmt" + "io" + "sync" + + "github.com/pierrec/lz4/v4/internal/lz4block" + "github.com/pierrec/lz4/v4/internal/lz4errors" + "github.com/pierrec/lz4/v4/internal/xxh32" +) + +type Blocks struct { + Block *FrameDataBlock + Blocks chan chan *FrameDataBlock + mu sync.Mutex + err error +} + +func (b *Blocks) initW(f *Frame, dst io.Writer, num int) { + if num == 1 { + b.Blocks = nil + b.Block = NewFrameDataBlock(f) + return + } + b.Block = nil + if cap(b.Blocks) != num { + b.Blocks = make(chan chan *FrameDataBlock, num) + } + // goroutine managing concurrent block compression goroutines. + go func() { + // Process next block compression item. + for c := range b.Blocks { + // Read the next compressed block result. + // Waiting here ensures that the blocks are output in the order they were sent. + // The incoming channel is always closed as it indicates to the caller that + // the block has been processed. + block := <-c + if block == nil { + // Notify the block compression routine that we are done with its result. + // This is used when a sentinel block is sent to terminate the compression. + close(c) + return + } + // Do not attempt to write the block upon any previous failure. + if b.err == nil { + // Write the block. + if err := block.Write(f, dst); err != nil { + // Keep the first error. + b.err = err + // All pending compression goroutines need to shut down, so we need to keep going. + } + } + close(c) + } + }() +} + +func (b *Blocks) close(f *Frame, num int) error { + if num == 1 { + if b.Block != nil { + b.Block.Close(f) + } + err := b.err + b.err = nil + return err + } + if b.Blocks == nil { + err := b.err + b.err = nil + return err + } + c := make(chan *FrameDataBlock) + b.Blocks <- c + c <- nil + <-c + err := b.err + b.err = nil + return err +} + +// ErrorR returns any error set while uncompressing a stream. +func (b *Blocks) ErrorR() error { + b.mu.Lock() + defer b.mu.Unlock() + return b.err +} + +// initR returns a channel that streams the uncompressed blocks if in concurrent +// mode and no error. When the channel is closed, check for any error with b.ErrorR. +// +// If not in concurrent mode, the uncompressed block is b.Block and the returned error +// needs to be checked. +func (b *Blocks) initR(f *Frame, num int, src io.Reader) (chan []byte, error) { + size := f.Descriptor.Flags.BlockSizeIndex() + if num == 1 { + b.Blocks = nil + b.Block = NewFrameDataBlock(f) + return nil, nil + } + b.Block = nil + blocks := make(chan chan []byte, num) + // data receives the uncompressed blocks. + data := make(chan []byte) + // Read blocks from the source sequentially + // and uncompress them concurrently. + + // In legacy mode, accrue the uncompress sizes in cum. + var cum uint32 + go func() { + var cumx uint32 + var err error + for b.ErrorR() == nil { + block := NewFrameDataBlock(f) + cumx, err = block.Read(f, src, 0) + if err != nil { + break + } + // Recheck for an error as reading may be slow and uncompressing is expensive. + if b.ErrorR() != nil { + break + } + c := make(chan []byte) + blocks <- c + go func() { + data, err := block.Uncompress(f, size.Get(), false) + if err != nil { + b.closeR(err) + } else { + c <- data + } + }() + } + // End the collection loop and the data channel. + c := make(chan []byte) + blocks <- c + c <- nil // signal the collection loop that we are done + <-c // wait for the collect loop to complete + if f.isLegacy() && cum == cumx { + err = io.EOF + } + b.closeR(err) + close(data) + }() + // Collect the uncompressed blocks and make them available + // on the returned channel. + go func(leg bool) { + defer close(blocks) + for c := range blocks { + buf := <-c + if buf == nil { + // Signal to end the loop. + close(c) + return + } + // Perform checksum now as the blocks are received in order. + if f.Descriptor.Flags.ContentChecksum() { + _, _ = f.checksum.Write(buf) + } + if leg { + cum += uint32(len(buf)) + } + data <- buf + close(c) + } + }(f.isLegacy()) + return data, nil +} + +// closeR safely sets the error on b if not already set. +func (b *Blocks) closeR(err error) { + b.mu.Lock() + if b.err == nil { + b.err = err + } + b.mu.Unlock() +} + +func NewFrameDataBlock(f *Frame) *FrameDataBlock { + buf := f.Descriptor.Flags.BlockSizeIndex().Get() + return &FrameDataBlock{Data: buf, data: buf} +} + +type FrameDataBlock struct { + Size DataBlockSize + Data []byte // compressed or uncompressed data (.data or .src) + Checksum uint32 + data []byte // buffer for compressed data + src []byte // uncompressed data + err error // used in concurrent mode +} + +func (b *FrameDataBlock) Close(f *Frame) { + b.Size = 0 + b.Checksum = 0 + b.err = nil + if b.data != nil { + // Block was not already closed. + lz4block.Put(b.data) + b.Data = nil + b.data = nil + b.src = nil + } +} + +// Block compression errors are ignored since the buffer is sized appropriately. +func (b *FrameDataBlock) Compress(f *Frame, src []byte, level lz4block.CompressionLevel) *FrameDataBlock { + data := b.data + if f.isLegacy() { + data = data[:cap(data)] + } else { + data = data[:len(src)] // trigger the incompressible flag in CompressBlock + } + var n int + switch level { + case lz4block.Fast: + n, _ = lz4block.CompressBlock(src, data) + default: + n, _ = lz4block.CompressBlockHC(src, data, level) + } + if n == 0 { + b.Size.UncompressedSet(true) + b.Data = src + } else { + b.Size.UncompressedSet(false) + b.Data = data[:n] + } + b.Size.sizeSet(len(b.Data)) + b.src = src // keep track of the source for content checksum + + if f.Descriptor.Flags.BlockChecksum() { + b.Checksum = xxh32.ChecksumZero(src) + } + return b +} + +func (b *FrameDataBlock) Write(f *Frame, dst io.Writer) error { + // Write is called in the same order as blocks are compressed, + // so content checksum must be done here. + if f.Descriptor.Flags.ContentChecksum() { + _, _ = f.checksum.Write(b.src) + } + buf := f.buf[:] + binary.LittleEndian.PutUint32(buf, uint32(b.Size)) + if _, err := dst.Write(buf[:4]); err != nil { + return err + } + + if _, err := dst.Write(b.Data); err != nil { + return err + } + + if b.Checksum == 0 { + return nil + } + binary.LittleEndian.PutUint32(buf, b.Checksum) + _, err := dst.Write(buf[:4]) + return err +} + +// Read updates b with the next block data, size and checksum if available. +func (b *FrameDataBlock) Read(f *Frame, src io.Reader, cum uint32) (uint32, error) { + x, err := f.readUint32(src) + if err != nil { + return 0, err + } + if f.isLegacy() { + switch x { + case frameMagicLegacy: + // Concatenated legacy frame. + return b.Read(f, src, cum) + case cum: + // Only works in non concurrent mode, for concurrent mode + // it is handled separately. + // Linux kernel format appends the total uncompressed size at the end. + return 0, io.EOF + } + } else if x == 0 { + // Marker for end of stream. + return 0, io.EOF + } + b.Size = DataBlockSize(x) + + size := b.Size.size() + if size > cap(b.data) { + return x, lz4errors.ErrOptionInvalidBlockSize + } + b.data = b.data[:size] + if _, err := io.ReadFull(src, b.data); err != nil { + return x, err + } + if f.Descriptor.Flags.BlockChecksum() { + sum, err := f.readUint32(src) + if err != nil { + return 0, err + } + b.Checksum = sum + } + return x, nil +} + +func (b *FrameDataBlock) Uncompress(f *Frame, dst []byte, sum bool) ([]byte, error) { + if b.Size.Uncompressed() { + n := copy(dst, b.data) + dst = dst[:n] + } else { + n, err := lz4block.UncompressBlock(b.data, dst) + if err != nil { + return nil, err + } + dst = dst[:n] + } + if f.Descriptor.Flags.BlockChecksum() { + if c := xxh32.ChecksumZero(dst); c != b.Checksum { + err := fmt.Errorf("%w: got %x; expected %x", lz4errors.ErrInvalidBlockChecksum, c, b.Checksum) + return nil, err + } + } + if sum && f.Descriptor.Flags.ContentChecksum() { + _, _ = f.checksum.Write(dst) + } + return dst, nil +} + +func (f *Frame) readUint32(r io.Reader) (x uint32, err error) { + if _, err = io.ReadFull(r, f.buf[:4]); err != nil { + return + } + x = binary.LittleEndian.Uint32(f.buf[:4]) + return +} diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go new file mode 100644 index 00000000..cfbd5674 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go @@ -0,0 +1,200 @@ +// Package lz4stream provides the types that support reading and writing LZ4 data streams. +package lz4stream + +import ( + "encoding/binary" + "fmt" + "io" + "io/ioutil" + + "github.com/pierrec/lz4/v4/internal/lz4block" + "github.com/pierrec/lz4/v4/internal/lz4errors" + "github.com/pierrec/lz4/v4/internal/xxh32" +) + +//go:generate go run gen.go + +const ( + frameMagic uint32 = 0x184D2204 + frameSkipMagic uint32 = 0x184D2A50 + frameMagicLegacy uint32 = 0x184C2102 +) + +func NewFrame() *Frame { + return &Frame{} +} + +type Frame struct { + buf [15]byte // frame descriptor needs at most 4(magic)+4+8+1=11 bytes + Magic uint32 + Descriptor FrameDescriptor + Blocks Blocks + Checksum uint32 + checksum xxh32.XXHZero +} + +// Reset allows reusing the Frame. +// The Descriptor configuration is not modified. +func (f *Frame) Reset(num int) { + f.Magic = 0 + f.Descriptor.Checksum = 0 + f.Descriptor.ContentSize = 0 + _ = f.Blocks.close(f, num) + f.Checksum = 0 +} + +func (f *Frame) InitW(dst io.Writer, num int, legacy bool) { + if legacy { + f.Magic = frameMagicLegacy + idx := lz4block.Index(lz4block.Block8Mb) + f.Descriptor.Flags.BlockSizeIndexSet(idx) + } else { + f.Magic = frameMagic + f.Descriptor.initW() + } + f.Blocks.initW(f, dst, num) + f.checksum.Reset() +} + +func (f *Frame) CloseW(dst io.Writer, num int) error { + if err := f.Blocks.close(f, num); err != nil { + return err + } + if f.isLegacy() { + return nil + } + buf := f.buf[:0] + // End mark (data block size of uint32(0)). + buf = append(buf, 0, 0, 0, 0) + if f.Descriptor.Flags.ContentChecksum() { + buf = f.checksum.Sum(buf) + } + _, err := dst.Write(buf) + return err +} + +func (f *Frame) isLegacy() bool { + return f.Magic == frameMagicLegacy +} + +func (f *Frame) InitR(src io.Reader, num int) (chan []byte, error) { + if f.Magic > 0 { + // Header already read. + return nil, nil + } + +newFrame: + var err error + if f.Magic, err = f.readUint32(src); err != nil { + return nil, err + } + switch m := f.Magic; { + case m == frameMagic || m == frameMagicLegacy: + // All 16 values of frameSkipMagic are valid. + case m>>8 == frameSkipMagic>>8: + skip, err := f.readUint32(src) + if err != nil { + return nil, err + } + if _, err := io.CopyN(ioutil.Discard, src, int64(skip)); err != nil { + return nil, err + } + goto newFrame + default: + return nil, lz4errors.ErrInvalidFrame + } + if err := f.Descriptor.initR(f, src); err != nil { + return nil, err + } + f.checksum.Reset() + return f.Blocks.initR(f, num, src) +} + +func (f *Frame) CloseR(src io.Reader) (err error) { + if f.isLegacy() { + return nil + } + if !f.Descriptor.Flags.ContentChecksum() { + return nil + } + if f.Checksum, err = f.readUint32(src); err != nil { + return err + } + if c := f.checksum.Sum32(); c != f.Checksum { + return fmt.Errorf("%w: got %x; expected %x", lz4errors.ErrInvalidFrameChecksum, c, f.Checksum) + } + return nil +} + +type FrameDescriptor struct { + Flags DescriptorFlags + ContentSize uint64 + Checksum uint8 +} + +func (fd *FrameDescriptor) initW() { + fd.Flags.VersionSet(1) + fd.Flags.BlockIndependenceSet(true) +} + +func (fd *FrameDescriptor) Write(f *Frame, dst io.Writer) error { + if fd.Checksum > 0 { + // Header already written. + return nil + } + + buf := f.buf[:4] + // Write the magic number here even though it belongs to the Frame. + binary.LittleEndian.PutUint32(buf, f.Magic) + if !f.isLegacy() { + buf = buf[:4+2] + binary.LittleEndian.PutUint16(buf[4:], uint16(fd.Flags)) + + if fd.Flags.Size() { + buf = buf[:4+2+8] + binary.LittleEndian.PutUint64(buf[4+2:], fd.ContentSize) + } + fd.Checksum = descriptorChecksum(buf[4:]) + buf = append(buf, fd.Checksum) + } + + _, err := dst.Write(buf) + return err +} + +func (fd *FrameDescriptor) initR(f *Frame, src io.Reader) error { + if f.isLegacy() { + idx := lz4block.Index(lz4block.Block8Mb) + f.Descriptor.Flags.BlockSizeIndexSet(idx) + return nil + } + // Read the flags and the checksum, hoping that there is not content size. + buf := f.buf[:3] + if _, err := io.ReadFull(src, buf); err != nil { + return err + } + descr := binary.LittleEndian.Uint16(buf) + fd.Flags = DescriptorFlags(descr) + if fd.Flags.Size() { + // Append the 8 missing bytes. + buf = buf[:3+8] + if _, err := io.ReadFull(src, buf[3:]); err != nil { + return err + } + fd.ContentSize = binary.LittleEndian.Uint64(buf[2:]) + } + fd.Checksum = buf[len(buf)-1] // the checksum is the last byte + buf = buf[:len(buf)-1] // all descriptor fields except checksum + if c := descriptorChecksum(buf); fd.Checksum != c { + return fmt.Errorf("%w: got %x; expected %x", lz4errors.ErrInvalidHeaderChecksum, c, fd.Checksum) + } + // Validate the elements that can be. + if idx := fd.Flags.BlockSizeIndex(); !idx.IsValid() { + return lz4errors.ErrOptionInvalidBlockSize + } + return nil +} + +func descriptorChecksum(buf []byte) byte { + return byte(xxh32.ChecksumZero(buf) >> 8) +} diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame_gen.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame_gen.go new file mode 100644 index 00000000..d33a6be9 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame_gen.go @@ -0,0 +1,103 @@ +// Code generated by `gen.exe`. DO NOT EDIT. + +package lz4stream + +import "github.com/pierrec/lz4/v4/internal/lz4block" + +// DescriptorFlags is defined as follow: +// field bits +// ----- ---- +// _ 2 +// ContentChecksum 1 +// Size 1 +// BlockChecksum 1 +// BlockIndependence 1 +// Version 2 +// _ 4 +// BlockSizeIndex 3 +// _ 1 +type DescriptorFlags uint16 + +// Getters. +func (x DescriptorFlags) ContentChecksum() bool { return x>>2&1 != 0 } +func (x DescriptorFlags) Size() bool { return x>>3&1 != 0 } +func (x DescriptorFlags) BlockChecksum() bool { return x>>4&1 != 0 } +func (x DescriptorFlags) BlockIndependence() bool { return x>>5&1 != 0 } +func (x DescriptorFlags) Version() uint16 { return uint16(x >> 6 & 0x3) } +func (x DescriptorFlags) BlockSizeIndex() lz4block.BlockSizeIndex { + return lz4block.BlockSizeIndex(x >> 12 & 0x7) +} + +// Setters. +func (x *DescriptorFlags) ContentChecksumSet(v bool) *DescriptorFlags { + const b = 1 << 2 + if v { + *x = *x&^b | b + } else { + *x &^= b + } + return x +} +func (x *DescriptorFlags) SizeSet(v bool) *DescriptorFlags { + const b = 1 << 3 + if v { + *x = *x&^b | b + } else { + *x &^= b + } + return x +} +func (x *DescriptorFlags) BlockChecksumSet(v bool) *DescriptorFlags { + const b = 1 << 4 + if v { + *x = *x&^b | b + } else { + *x &^= b + } + return x +} +func (x *DescriptorFlags) BlockIndependenceSet(v bool) *DescriptorFlags { + const b = 1 << 5 + if v { + *x = *x&^b | b + } else { + *x &^= b + } + return x +} +func (x *DescriptorFlags) VersionSet(v uint16) *DescriptorFlags { + *x = *x&^(0x3<<6) | (DescriptorFlags(v) & 0x3 << 6) + return x +} +func (x *DescriptorFlags) BlockSizeIndexSet(v lz4block.BlockSizeIndex) *DescriptorFlags { + *x = *x&^(0x7<<12) | (DescriptorFlags(v) & 0x7 << 12) + return x +} + +// Code generated by `gen.exe`. DO NOT EDIT. + +// DataBlockSize is defined as follow: +// field bits +// ----- ---- +// size 31 +// Uncompressed 1 +type DataBlockSize uint32 + +// Getters. +func (x DataBlockSize) size() int { return int(x & 0x7FFFFFFF) } +func (x DataBlockSize) Uncompressed() bool { return x>>31&1 != 0 } + +// Setters. +func (x *DataBlockSize) sizeSet(v int) *DataBlockSize { + *x = *x&^0x7FFFFFFF | DataBlockSize(v)&0x7FFFFFFF + return x +} +func (x *DataBlockSize) UncompressedSet(v bool) *DataBlockSize { + const b = 1 << 31 + if v { + *x = *x&^b | b + } else { + *x &^= b + } + return x +} diff --git a/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero.go b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero.go new file mode 100644 index 00000000..8d3206a8 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero.go @@ -0,0 +1,212 @@ +// Package xxh32 implements the very fast XXH hashing algorithm (32 bits version). +// (https://github.com/Cyan4973/XXH/) +package xxh32 + +import ( + "encoding/binary" +) + +const ( + prime1 uint32 = 2654435761 + prime2 uint32 = 2246822519 + prime3 uint32 = 3266489917 + prime4 uint32 = 668265263 + prime5 uint32 = 374761393 + + primeMask = 0xFFFFFFFF + prime1plus2 = uint32((uint64(prime1) + uint64(prime2)) & primeMask) // 606290984 + prime1minus = uint32((-int64(prime1)) & primeMask) // 1640531535 +) + +// XXHZero represents an xxhash32 object with seed 0. +type XXHZero struct { + v [4]uint32 + totalLen uint64 + buf [16]byte + bufused int +} + +// Sum appends the current hash to b and returns the resulting slice. +// It does not change the underlying hash state. +func (xxh XXHZero) Sum(b []byte) []byte { + h32 := xxh.Sum32() + return append(b, byte(h32), byte(h32>>8), byte(h32>>16), byte(h32>>24)) +} + +// Reset resets the Hash to its initial state. +func (xxh *XXHZero) Reset() { + xxh.v[0] = prime1plus2 + xxh.v[1] = prime2 + xxh.v[2] = 0 + xxh.v[3] = prime1minus + xxh.totalLen = 0 + xxh.bufused = 0 +} + +// Size returns the number of bytes returned by Sum(). +func (xxh *XXHZero) Size() int { + return 4 +} + +// BlockSizeIndex gives the minimum number of bytes accepted by Write(). +func (xxh *XXHZero) BlockSize() int { + return 1 +} + +// Write adds input bytes to the Hash. +// It never returns an error. +func (xxh *XXHZero) Write(input []byte) (int, error) { + if xxh.totalLen == 0 { + xxh.Reset() + } + n := len(input) + m := xxh.bufused + + xxh.totalLen += uint64(n) + + r := len(xxh.buf) - m + if n < r { + copy(xxh.buf[m:], input) + xxh.bufused += len(input) + return n, nil + } + + var buf *[16]byte + if m != 0 { + // some data left from previous update + buf = &xxh.buf + c := copy(buf[m:], input) + n -= c + input = input[c:] + } + update(&xxh.v, buf, input) + xxh.bufused = copy(xxh.buf[:], input[n-n%16:]) + + return n, nil +} + +// Portable version of update. This updates v by processing all of buf +// (if not nil) and all full 16-byte blocks of input. +func updateGo(v *[4]uint32, buf *[16]byte, input []byte) { + // Causes compiler to work directly from registers instead of stack: + v1, v2, v3, v4 := v[0], v[1], v[2], v[3] + + if buf != nil { + v1 = rol13(v1+binary.LittleEndian.Uint32(buf[:])*prime2) * prime1 + v2 = rol13(v2+binary.LittleEndian.Uint32(buf[4:])*prime2) * prime1 + v3 = rol13(v3+binary.LittleEndian.Uint32(buf[8:])*prime2) * prime1 + v4 = rol13(v4+binary.LittleEndian.Uint32(buf[12:])*prime2) * prime1 + } + + for ; len(input) >= 16; input = input[16:] { + sub := input[:16] //BCE hint for compiler + v1 = rol13(v1+binary.LittleEndian.Uint32(sub[:])*prime2) * prime1 + v2 = rol13(v2+binary.LittleEndian.Uint32(sub[4:])*prime2) * prime1 + v3 = rol13(v3+binary.LittleEndian.Uint32(sub[8:])*prime2) * prime1 + v4 = rol13(v4+binary.LittleEndian.Uint32(sub[12:])*prime2) * prime1 + } + v[0], v[1], v[2], v[3] = v1, v2, v3, v4 +} + +// Sum32 returns the 32 bits Hash value. +func (xxh *XXHZero) Sum32() uint32 { + h32 := uint32(xxh.totalLen) + if h32 >= 16 { + h32 += rol1(xxh.v[0]) + rol7(xxh.v[1]) + rol12(xxh.v[2]) + rol18(xxh.v[3]) + } else { + h32 += prime5 + } + + p := 0 + n := xxh.bufused + buf := xxh.buf + for n := n - 4; p <= n; p += 4 { + h32 += binary.LittleEndian.Uint32(buf[p:p+4]) * prime3 + h32 = rol17(h32) * prime4 + } + for ; p < n; p++ { + h32 += uint32(buf[p]) * prime5 + h32 = rol11(h32) * prime1 + } + + h32 ^= h32 >> 15 + h32 *= prime2 + h32 ^= h32 >> 13 + h32 *= prime3 + h32 ^= h32 >> 16 + + return h32 +} + +// Portable version of ChecksumZero. +func checksumZeroGo(input []byte) uint32 { + n := len(input) + h32 := uint32(n) + + if n < 16 { + h32 += prime5 + } else { + v1 := prime1plus2 + v2 := prime2 + v3 := uint32(0) + v4 := prime1minus + p := 0 + for n := n - 16; p <= n; p += 16 { + sub := input[p:][:16] //BCE hint for compiler + v1 = rol13(v1+binary.LittleEndian.Uint32(sub[:])*prime2) * prime1 + v2 = rol13(v2+binary.LittleEndian.Uint32(sub[4:])*prime2) * prime1 + v3 = rol13(v3+binary.LittleEndian.Uint32(sub[8:])*prime2) * prime1 + v4 = rol13(v4+binary.LittleEndian.Uint32(sub[12:])*prime2) * prime1 + } + input = input[p:] + n -= p + h32 += rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) + } + + p := 0 + for n := n - 4; p <= n; p += 4 { + h32 += binary.LittleEndian.Uint32(input[p:p+4]) * prime3 + h32 = rol17(h32) * prime4 + } + for p < n { + h32 += uint32(input[p]) * prime5 + h32 = rol11(h32) * prime1 + p++ + } + + h32 ^= h32 >> 15 + h32 *= prime2 + h32 ^= h32 >> 13 + h32 *= prime3 + h32 ^= h32 >> 16 + + return h32 +} + +func rol1(u uint32) uint32 { + return u<<1 | u>>31 +} + +func rol7(u uint32) uint32 { + return u<<7 | u>>25 +} + +func rol11(u uint32) uint32 { + return u<<11 | u>>21 +} + +func rol12(u uint32) uint32 { + return u<<12 | u>>20 +} + +func rol13(u uint32) uint32 { + return u<<13 | u>>19 +} + +func rol17(u uint32) uint32 { + return u<<17 | u>>15 +} + +func rol18(u uint32) uint32 { + return u<<18 | u>>14 +} diff --git a/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.go b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.go new file mode 100644 index 00000000..0978b266 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.go @@ -0,0 +1,11 @@ +// +build !noasm + +package xxh32 + +// ChecksumZero returns the 32-bit hash of input. +// +//go:noescape +func ChecksumZero(input []byte) uint32 + +//go:noescape +func update(v *[4]uint32, buf *[16]byte, input []byte) diff --git a/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.s b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.s new file mode 100644 index 00000000..0e9f146a --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.s @@ -0,0 +1,259 @@ +// +build !noasm + +#include "textflag.h" + +#define prime1 $2654435761 +#define prime2 $2246822519 +#define prime3 $3266489917 +#define prime4 $668265263 +#define prime5 $374761393 + +#define prime1plus2 $606290984 +#define prime1minus $1640531535 + +// Register allocation. +#define p R0 +#define n R1 +#define h R2 +#define v1 R2 // Alias for h. +#define v2 R3 +#define v3 R4 +#define v4 R5 +#define x1 R6 +#define x2 R7 +#define x3 R8 +#define x4 R9 + +// We need the primes in registers. The 16-byte loop only uses prime{1,2}. +#define prime1r R11 +#define prime2r R12 +#define prime3r R3 // The rest can alias v{2-4}. +#define prime4r R4 +#define prime5r R5 + +// Update round macros. These read from and increment p. + +#define round16aligned \ + MOVM.IA.W (p), [x1, x2, x3, x4] \ + \ + MULA x1, prime2r, v1, v1 \ + MULA x2, prime2r, v2, v2 \ + MULA x3, prime2r, v3, v3 \ + MULA x4, prime2r, v4, v4 \ + \ + MOVW v1 @> 19, v1 \ + MOVW v2 @> 19, v2 \ + MOVW v3 @> 19, v3 \ + MOVW v4 @> 19, v4 \ + \ + MUL prime1r, v1 \ + MUL prime1r, v2 \ + MUL prime1r, v3 \ + MUL prime1r, v4 \ + +#define round16unaligned \ + MOVBU.P 16(p), x1 \ + MOVBU -15(p), x2 \ + ORR x2 << 8, x1 \ + MOVBU -14(p), x3 \ + MOVBU -13(p), x4 \ + ORR x4 << 8, x3 \ + ORR x3 << 16, x1 \ + \ + MULA x1, prime2r, v1, v1 \ + MOVW v1 @> 19, v1 \ + MUL prime1r, v1 \ + \ + MOVBU -12(p), x1 \ + MOVBU -11(p), x2 \ + ORR x2 << 8, x1 \ + MOVBU -10(p), x3 \ + MOVBU -9(p), x4 \ + ORR x4 << 8, x3 \ + ORR x3 << 16, x1 \ + \ + MULA x1, prime2r, v2, v2 \ + MOVW v2 @> 19, v2 \ + MUL prime1r, v2 \ + \ + MOVBU -8(p), x1 \ + MOVBU -7(p), x2 \ + ORR x2 << 8, x1 \ + MOVBU -6(p), x3 \ + MOVBU -5(p), x4 \ + ORR x4 << 8, x3 \ + ORR x3 << 16, x1 \ + \ + MULA x1, prime2r, v3, v3 \ + MOVW v3 @> 19, v3 \ + MUL prime1r, v3 \ + \ + MOVBU -4(p), x1 \ + MOVBU -3(p), x2 \ + ORR x2 << 8, x1 \ + MOVBU -2(p), x3 \ + MOVBU -1(p), x4 \ + ORR x4 << 8, x3 \ + ORR x3 << 16, x1 \ + \ + MULA x1, prime2r, v4, v4 \ + MOVW v4 @> 19, v4 \ + MUL prime1r, v4 \ + + +// func ChecksumZero([]byte) uint32 +TEXT ·ChecksumZero(SB), NOFRAME|NOSPLIT, $-4-16 + MOVW input_base+0(FP), p + MOVW input_len+4(FP), n + + MOVW prime1, prime1r + MOVW prime2, prime2r + + // Set up h for n < 16. It's tempting to say {ADD prime5, n, h} + // here, but that's a pseudo-op that generates a load through R11. + MOVW prime5, prime5r + ADD prime5r, n, h + CMP $0, n + BEQ end + + // We let n go negative so we can do comparisons with SUB.S + // instead of separate CMP. + SUB.S $16, n + BMI loop16done + + MOVW prime1plus2, v1 + MOVW prime2, v2 + MOVW $0, v3 + MOVW prime1minus, v4 + + TST $3, p + BNE loop16unaligned + +loop16aligned: + SUB.S $16, n + round16aligned + BPL loop16aligned + B loop16finish + +loop16unaligned: + SUB.S $16, n + round16unaligned + BPL loop16unaligned + +loop16finish: + MOVW v1 @> 31, h + ADD v2 @> 25, h + ADD v3 @> 20, h + ADD v4 @> 14, h + + // h += len(input) with v2 as temporary. + MOVW input_len+4(FP), v2 + ADD v2, h + +loop16done: + ADD $16, n // Restore number of bytes left. + + SUB.S $4, n + MOVW prime3, prime3r + BMI loop4done + MOVW prime4, prime4r + + TST $3, p + BNE loop4unaligned + +loop4aligned: + SUB.S $4, n + + MOVW.P 4(p), x1 + MULA prime3r, x1, h, h + MOVW h @> 15, h + MUL prime4r, h + + BPL loop4aligned + B loop4done + +loop4unaligned: + SUB.S $4, n + + MOVBU.P 4(p), x1 + MOVBU -3(p), x2 + ORR x2 << 8, x1 + MOVBU -2(p), x3 + ORR x3 << 16, x1 + MOVBU -1(p), x4 + ORR x4 << 24, x1 + + MULA prime3r, x1, h, h + MOVW h @> 15, h + MUL prime4r, h + + BPL loop4unaligned + +loop4done: + ADD.S $4, n // Restore number of bytes left. + BEQ end + + MOVW prime5, prime5r + +loop1: + SUB.S $1, n + + MOVBU.P 1(p), x1 + MULA prime5r, x1, h, h + MOVW h @> 21, h + MUL prime1r, h + + BNE loop1 + +end: + MOVW prime3, prime3r + EOR h >> 15, h + MUL prime2r, h + EOR h >> 13, h + MUL prime3r, h + EOR h >> 16, h + + MOVW h, ret+12(FP) + RET + + +// func update(v *[4]uint64, buf *[16]byte, p []byte) +TEXT ·update(SB), NOFRAME|NOSPLIT, $-4-20 + MOVW v+0(FP), p + MOVM.IA (p), [v1, v2, v3, v4] + + MOVW prime1, prime1r + MOVW prime2, prime2r + + // Process buf, if not nil. + MOVW buf+4(FP), p + CMP $0, p + BEQ noBuffered + + round16aligned + +noBuffered: + MOVW input_base +8(FP), p + MOVW input_len +12(FP), n + + SUB.S $16, n + BMI end + + TST $3, p + BNE loop16unaligned + +loop16aligned: + SUB.S $16, n + round16aligned + BPL loop16aligned + B end + +loop16unaligned: + SUB.S $16, n + round16unaligned + BPL loop16unaligned + +end: + MOVW v+0(FP), p + MOVM.IA [v1, v2, v3, v4], (p) + RET diff --git a/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_other.go b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_other.go new file mode 100644 index 00000000..c96b59b8 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_other.go @@ -0,0 +1,10 @@ +// +build !arm noasm + +package xxh32 + +// ChecksumZero returns the 32-bit hash of input. +func ChecksumZero(input []byte) uint32 { return checksumZeroGo(input) } + +func update(v *[4]uint32, buf *[16]byte, input []byte) { + updateGo(v, buf, input) +} diff --git a/vendor/github.com/pierrec/lz4/v4/lz4.go b/vendor/github.com/pierrec/lz4/v4/lz4.go new file mode 100644 index 00000000..c585d406 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/lz4.go @@ -0,0 +1,147 @@ +// Package lz4 implements reading and writing lz4 compressed data. +// +// The package supports both the LZ4 stream format, +// as specified in http://fastcompression.blogspot.fr/2013/04/lz4-streaming-format-final.html, +// and the LZ4 block format, defined at +// http://fastcompression.blogspot.fr/2011/05/lz4-explained.html. +// +// See https://github.com/lz4/lz4 for the reference C implementation. +package lz4 + +import ( + "github.com/pierrec/lz4/v4/internal/lz4block" + "github.com/pierrec/lz4/v4/internal/lz4errors" +) + +func _() { + // Safety checks for duplicated elements. + var x [1]struct{} + _ = x[lz4block.CompressionLevel(Fast)-lz4block.Fast] + _ = x[Block64Kb-BlockSize(lz4block.Block64Kb)] + _ = x[Block256Kb-BlockSize(lz4block.Block256Kb)] + _ = x[Block1Mb-BlockSize(lz4block.Block1Mb)] + _ = x[Block4Mb-BlockSize(lz4block.Block4Mb)] +} + +// CompressBlockBound returns the maximum size of a given buffer of size n, when not compressible. +func CompressBlockBound(n int) int { + return lz4block.CompressBlockBound(n) +} + +// UncompressBlock uncompresses the source buffer into the destination one, +// and returns the uncompressed size. +// +// The destination buffer must be sized appropriately. +// +// An error is returned if the source data is invalid or the destination buffer is too small. +func UncompressBlock(src, dst []byte) (int, error) { + return lz4block.UncompressBlock(src, dst) +} + +// A Compressor compresses data into the LZ4 block format. +// It uses a fast compression algorithm. +// +// A Compressor is not safe for concurrent use by multiple goroutines. +// +// Use a Writer to compress into the LZ4 stream format. +type Compressor struct{ c lz4block.Compressor } + +// CompressBlock compresses the source buffer src into the destination dst. +// +// If compression is successful, the first return value is the size of the +// compressed data, which is always >0. +// +// If dst has length at least CompressBlockBound(len(src)), compression always +// succeeds. Otherwise, the first return value is zero. The error return is +// non-nil if the compressed data does not fit in dst, but it might fit in a +// larger buffer that is still smaller than CompressBlockBound(len(src)). The +// return value (0, nil) means the data is likely incompressible and a buffer +// of length CompressBlockBound(len(src)) should be passed in. +func (c *Compressor) CompressBlock(src, dst []byte) (int, error) { + return c.c.CompressBlock(src, dst) +} + +// CompressBlock compresses the source buffer into the destination one. +// This is the fast version of LZ4 compression and also the default one. +// +// The argument hashTable is scratch space for a hash table used by the +// compressor. If provided, it should have length at least 1<<16. If it is +// shorter (or nil), CompressBlock allocates its own hash table. +// +// The size of the compressed data is returned. +// +// If the destination buffer size is lower than CompressBlockBound and +// the compressed size is 0 and no error, then the data is incompressible. +// +// An error is returned if the destination buffer is too small. + +// CompressBlock is equivalent to Compressor.CompressBlock. +// The final argument is ignored and should be set to nil. +// +// This function is deprecated. Use a Compressor instead. +func CompressBlock(src, dst []byte, _ []int) (int, error) { + return lz4block.CompressBlock(src, dst) +} + +// A CompressorHC compresses data into the LZ4 block format. +// Its compression ratio is potentially better than that of a Compressor, +// but it is also slower and requires more memory. +// +// A Compressor is not safe for concurrent use by multiple goroutines. +// +// Use a Writer to compress into the LZ4 stream format. +type CompressorHC struct { + // Level is the maximum search depth for compression. + // Values <= 0 mean no maximum. + Level CompressionLevel + c lz4block.CompressorHC +} + +// CompressBlock compresses the source buffer src into the destination dst. +// +// If compression is successful, the first return value is the size of the +// compressed data, which is always >0. +// +// If dst has length at least CompressBlockBound(len(src)), compression always +// succeeds. Otherwise, the first return value is zero. The error return is +// non-nil if the compressed data does not fit in dst, but it might fit in a +// larger buffer that is still smaller than CompressBlockBound(len(src)). The +// return value (0, nil) means the data is likely incompressible and a buffer +// of length CompressBlockBound(len(src)) should be passed in. +func (c *CompressorHC) CompressBlock(src, dst []byte) (int, error) { + return c.c.CompressBlock(src, dst, lz4block.CompressionLevel(c.Level)) +} + +// CompressBlockHC is equivalent to CompressorHC.CompressBlock. +// The final two arguments are ignored and should be set to nil. +// +// This function is deprecated. Use a CompressorHC instead. +func CompressBlockHC(src, dst []byte, depth CompressionLevel, _, _ []int) (int, error) { + return lz4block.CompressBlockHC(src, dst, lz4block.CompressionLevel(depth)) +} + +const ( + // ErrInvalidSourceShortBuffer is returned by UncompressBlock or CompressBLock when a compressed + // block is corrupted or the destination buffer is not large enough for the uncompressed data. + ErrInvalidSourceShortBuffer = lz4errors.ErrInvalidSourceShortBuffer + // ErrInvalidFrame is returned when reading an invalid LZ4 archive. + ErrInvalidFrame = lz4errors.ErrInvalidFrame + // ErrInternalUnhandledState is an internal error. + ErrInternalUnhandledState = lz4errors.ErrInternalUnhandledState + // ErrInvalidHeaderChecksum is returned when reading a frame. + ErrInvalidHeaderChecksum = lz4errors.ErrInvalidHeaderChecksum + // ErrInvalidBlockChecksum is returned when reading a frame. + ErrInvalidBlockChecksum = lz4errors.ErrInvalidBlockChecksum + // ErrInvalidFrameChecksum is returned when reading a frame. + ErrInvalidFrameChecksum = lz4errors.ErrInvalidFrameChecksum + // ErrOptionInvalidCompressionLevel is returned when the supplied compression level is invalid. + ErrOptionInvalidCompressionLevel = lz4errors.ErrOptionInvalidCompressionLevel + // ErrOptionClosedOrError is returned when an option is applied to a closed or in error object. + ErrOptionClosedOrError = lz4errors.ErrOptionClosedOrError + // ErrOptionInvalidBlockSize is returned when + ErrOptionInvalidBlockSize = lz4errors.ErrOptionInvalidBlockSize + // ErrOptionNotApplicable is returned when trying to apply an option to an object not supporting it. + ErrOptionNotApplicable = lz4errors.ErrOptionNotApplicable + // ErrWriterNotClosed is returned when attempting to reset an unclosed writer. + ErrWriterNotClosed = lz4errors.ErrWriterNotClosed +) diff --git a/vendor/github.com/pierrec/lz4/v4/options.go b/vendor/github.com/pierrec/lz4/v4/options.go new file mode 100644 index 00000000..4e1b6703 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/options.go @@ -0,0 +1,213 @@ +package lz4 + +import ( + "fmt" + "github.com/pierrec/lz4/v4/internal/lz4block" + "github.com/pierrec/lz4/v4/internal/lz4errors" + "reflect" + "runtime" +) + +//go:generate go run golang.org/x/tools/cmd/stringer -type=BlockSize,CompressionLevel -output options_gen.go + +type ( + applier interface { + Apply(...Option) error + private() + } + // Option defines the parameters to setup an LZ4 Writer or Reader. + Option func(applier) error +) + +// String returns a string representation of the option with its parameter(s). +func (o Option) String() string { + return o(nil).Error() +} + +// Default options. +var ( + DefaultBlockSizeOption = BlockSizeOption(Block4Mb) + DefaultChecksumOption = ChecksumOption(true) + DefaultConcurrency = ConcurrencyOption(1) + defaultOnBlockDone = OnBlockDoneOption(nil) +) + +const ( + Block64Kb BlockSize = 1 << (16 + iota*2) + Block256Kb + Block1Mb + Block4Mb +) + +// BlockSizeIndex defines the size of the blocks to be compressed. +type BlockSize uint32 + +// BlockSizeOption defines the maximum size of compressed blocks (default=Block4Mb). +func BlockSizeOption(size BlockSize) Option { + return func(a applier) error { + switch w := a.(type) { + case nil: + s := fmt.Sprintf("BlockSizeOption(%s)", size) + return lz4errors.Error(s) + case *Writer: + size := uint32(size) + if !lz4block.IsValid(size) { + return fmt.Errorf("%w: %d", lz4errors.ErrOptionInvalidBlockSize, size) + } + w.frame.Descriptor.Flags.BlockSizeIndexSet(lz4block.Index(size)) + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} + +// BlockChecksumOption enables or disables block checksum (default=false). +func BlockChecksumOption(flag bool) Option { + return func(a applier) error { + switch w := a.(type) { + case nil: + s := fmt.Sprintf("BlockChecksumOption(%v)", flag) + return lz4errors.Error(s) + case *Writer: + w.frame.Descriptor.Flags.BlockChecksumSet(flag) + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} + +// ChecksumOption enables/disables all blocks or content checksum (default=true). +func ChecksumOption(flag bool) Option { + return func(a applier) error { + switch w := a.(type) { + case nil: + s := fmt.Sprintf("ChecksumOption(%v)", flag) + return lz4errors.Error(s) + case *Writer: + w.frame.Descriptor.Flags.ContentChecksumSet(flag) + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} + +// SizeOption sets the size of the original uncompressed data (default=0). It is useful to know the size of the +// whole uncompressed data stream. +func SizeOption(size uint64) Option { + return func(a applier) error { + switch w := a.(type) { + case nil: + s := fmt.Sprintf("SizeOption(%d)", size) + return lz4errors.Error(s) + case *Writer: + w.frame.Descriptor.Flags.SizeSet(size > 0) + w.frame.Descriptor.ContentSize = size + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} + +// ConcurrencyOption sets the number of go routines used for compression. +// If n <= 0, then the output of runtime.GOMAXPROCS(0) is used. +func ConcurrencyOption(n int) Option { + if n <= 0 { + n = runtime.GOMAXPROCS(0) + } + return func(a applier) error { + switch rw := a.(type) { + case nil: + s := fmt.Sprintf("ConcurrencyOption(%d)", n) + return lz4errors.Error(s) + case *Writer: + rw.num = n + return nil + case *Reader: + rw.num = n + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} + +// CompressionLevel defines the level of compression to use. The higher the better, but slower, compression. +type CompressionLevel uint32 + +const ( + Fast CompressionLevel = 0 + Level1 CompressionLevel = 1 << (8 + iota) + Level2 + Level3 + Level4 + Level5 + Level6 + Level7 + Level8 + Level9 +) + +// CompressionLevelOption defines the compression level (default=Fast). +func CompressionLevelOption(level CompressionLevel) Option { + return func(a applier) error { + switch w := a.(type) { + case nil: + s := fmt.Sprintf("CompressionLevelOption(%s)", level) + return lz4errors.Error(s) + case *Writer: + switch level { + case Fast, Level1, Level2, Level3, Level4, Level5, Level6, Level7, Level8, Level9: + default: + return fmt.Errorf("%w: %d", lz4errors.ErrOptionInvalidCompressionLevel, level) + } + w.level = lz4block.CompressionLevel(level) + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} + +func onBlockDone(int) {} + +// OnBlockDoneOption is triggered when a block has been processed. For a Writer, it is when is has been compressed, +// for a Reader, it is when it has been uncompressed. +func OnBlockDoneOption(handler func(size int)) Option { + if handler == nil { + handler = onBlockDone + } + return func(a applier) error { + switch rw := a.(type) { + case nil: + s := fmt.Sprintf("OnBlockDoneOption(%s)", reflect.TypeOf(handler).String()) + return lz4errors.Error(s) + case *Writer: + rw.handler = handler + return nil + case *Reader: + rw.handler = handler + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} + +// LegacyOption provides support for writing LZ4 frames in the legacy format. +// +// See https://github.com/lz4/lz4/blob/dev/doc/lz4_Frame_format.md#legacy-frame. +// +// NB. compressed Linux kernel images use a tweaked LZ4 legacy format where +// the compressed stream is followed by the original (uncompressed) size of +// the kernel (https://events.static.linuxfound.org/sites/events/files/lcjpcojp13_klee.pdf). +// This is also supported as a special case. +func LegacyOption(legacy bool) Option { + return func(a applier) error { + switch rw := a.(type) { + case nil: + s := fmt.Sprintf("LegacyOption(%v)", legacy) + return lz4errors.Error(s) + case *Writer: + rw.legacy = legacy + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} diff --git a/vendor/github.com/pierrec/lz4/v4/options_gen.go b/vendor/github.com/pierrec/lz4/v4/options_gen.go new file mode 100644 index 00000000..2de81490 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/options_gen.go @@ -0,0 +1,92 @@ +// Code generated by "stringer -type=BlockSize,CompressionLevel -output options_gen.go"; DO NOT EDIT. + +package lz4 + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Block64Kb-65536] + _ = x[Block256Kb-262144] + _ = x[Block1Mb-1048576] + _ = x[Block4Mb-4194304] +} + +const ( + _BlockSize_name_0 = "Block64Kb" + _BlockSize_name_1 = "Block256Kb" + _BlockSize_name_2 = "Block1Mb" + _BlockSize_name_3 = "Block4Mb" +) + +func (i BlockSize) String() string { + switch { + case i == 65536: + return _BlockSize_name_0 + case i == 262144: + return _BlockSize_name_1 + case i == 1048576: + return _BlockSize_name_2 + case i == 4194304: + return _BlockSize_name_3 + default: + return "BlockSize(" + strconv.FormatInt(int64(i), 10) + ")" + } +} +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Fast-0] + _ = x[Level1-512] + _ = x[Level2-1024] + _ = x[Level3-2048] + _ = x[Level4-4096] + _ = x[Level5-8192] + _ = x[Level6-16384] + _ = x[Level7-32768] + _ = x[Level8-65536] + _ = x[Level9-131072] +} + +const ( + _CompressionLevel_name_0 = "Fast" + _CompressionLevel_name_1 = "Level1" + _CompressionLevel_name_2 = "Level2" + _CompressionLevel_name_3 = "Level3" + _CompressionLevel_name_4 = "Level4" + _CompressionLevel_name_5 = "Level5" + _CompressionLevel_name_6 = "Level6" + _CompressionLevel_name_7 = "Level7" + _CompressionLevel_name_8 = "Level8" + _CompressionLevel_name_9 = "Level9" +) + +func (i CompressionLevel) String() string { + switch { + case i == 0: + return _CompressionLevel_name_0 + case i == 512: + return _CompressionLevel_name_1 + case i == 1024: + return _CompressionLevel_name_2 + case i == 2048: + return _CompressionLevel_name_3 + case i == 4096: + return _CompressionLevel_name_4 + case i == 8192: + return _CompressionLevel_name_5 + case i == 16384: + return _CompressionLevel_name_6 + case i == 32768: + return _CompressionLevel_name_7 + case i == 65536: + return _CompressionLevel_name_8 + case i == 131072: + return _CompressionLevel_name_9 + default: + return "CompressionLevel(" + strconv.FormatInt(int64(i), 10) + ")" + } +} diff --git a/vendor/github.com/pierrec/lz4/v4/reader.go b/vendor/github.com/pierrec/lz4/v4/reader.go new file mode 100644 index 00000000..403aaf69 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/reader.go @@ -0,0 +1,243 @@ +package lz4 + +import ( + "io" + + "github.com/pierrec/lz4/v4/internal/lz4block" + "github.com/pierrec/lz4/v4/internal/lz4errors" + "github.com/pierrec/lz4/v4/internal/lz4stream" +) + +var readerStates = []aState{ + noState: newState, + errorState: newState, + newState: readState, + readState: closedState, + closedState: newState, +} + +// NewReader returns a new LZ4 frame decoder. +func NewReader(r io.Reader) *Reader { + return newReader(r, false) +} + +func newReader(r io.Reader, legacy bool) *Reader { + zr := &Reader{frame: lz4stream.NewFrame()} + zr.state.init(readerStates) + _ = zr.Apply(DefaultConcurrency, defaultOnBlockDone) + zr.Reset(r) + return zr +} + +// Reader allows reading an LZ4 stream. +type Reader struct { + state _State + src io.Reader // source reader + num int // concurrency level + frame *lz4stream.Frame // frame being read + data []byte // block buffer allocated in non concurrent mode + reads chan []byte // pending data + idx int // size of pending data + handler func(int) + cum uint32 +} + +func (*Reader) private() {} + +func (r *Reader) Apply(options ...Option) (err error) { + defer r.state.check(&err) + switch r.state.state { + case newState: + case errorState: + return r.state.err + default: + return lz4errors.ErrOptionClosedOrError + } + for _, o := range options { + if err = o(r); err != nil { + return + } + } + return +} + +// Size returns the size of the underlying uncompressed data, if set in the stream. +func (r *Reader) Size() int { + switch r.state.state { + case readState, closedState: + if r.frame.Descriptor.Flags.Size() { + return int(r.frame.Descriptor.ContentSize) + } + } + return 0 +} + +func (r *Reader) isNotConcurrent() bool { + return r.num == 1 +} + +func (r *Reader) init() error { + data, err := r.frame.InitR(r.src, r.num) + if err != nil { + return err + } + r.reads = data + r.idx = 0 + size := r.frame.Descriptor.Flags.BlockSizeIndex() + r.data = size.Get() + r.cum = 0 + return nil +} + +func (r *Reader) Read(buf []byte) (n int, err error) { + defer r.state.check(&err) + switch r.state.state { + case readState: + case closedState, errorState: + return 0, r.state.err + case newState: + // First initialization. + if err = r.init(); r.state.next(err) { + return + } + default: + return 0, r.state.fail() + } + for len(buf) > 0 { + var bn int + if r.idx == 0 { + if r.isNotConcurrent() { + bn, err = r.read(buf) + } else { + lz4block.Put(r.data) + r.data = <-r.reads + if len(r.data) == 0 { + // No uncompressed data: something went wrong or we are done. + err = r.frame.Blocks.ErrorR() + } + } + switch err { + case nil: + case io.EOF: + if er := r.frame.CloseR(r.src); er != nil { + err = er + } + lz4block.Put(r.data) + r.data = nil + return + default: + return + } + } + if bn == 0 { + // Fill buf with buffered data. + bn = copy(buf, r.data[r.idx:]) + r.idx += bn + if r.idx == len(r.data) { + // All data read, get ready for the next Read. + r.idx = 0 + } + } + buf = buf[bn:] + n += bn + r.handler(bn) + } + return +} + +// read uncompresses the next block as follow: +// - if buf has enough room, the block is uncompressed into it directly +// and the lenght of used space is returned +// - else, the uncompress data is stored in r.data and 0 is returned +func (r *Reader) read(buf []byte) (int, error) { + block := r.frame.Blocks.Block + _, err := block.Read(r.frame, r.src, r.cum) + if err != nil { + return 0, err + } + var direct bool + dst := r.data[:cap(r.data)] + if len(buf) >= len(dst) { + // Uncompress directly into buf. + direct = true + dst = buf + } + dst, err = block.Uncompress(r.frame, dst, true) + if err != nil { + return 0, err + } + r.cum += uint32(len(dst)) + if direct { + return len(dst), nil + } + r.data = dst + return 0, nil +} + +// Reset clears the state of the Reader r such that it is equivalent to its +// initial state from NewReader, but instead writing to writer. +// No access to reader is performed. +// +// w.Close must be called before Reset. +func (r *Reader) Reset(reader io.Reader) { + if r.data != nil { + lz4block.Put(r.data) + r.data = nil + } + r.frame.Reset(r.num) + r.state.reset() + r.src = reader + r.reads = nil +} + +// WriteTo efficiently uncompresses the data from the Reader underlying source to w. +func (r *Reader) WriteTo(w io.Writer) (n int64, err error) { + switch r.state.state { + case closedState, errorState: + return 0, r.state.err + case newState: + if err = r.init(); r.state.next(err) { + return + } + default: + return 0, r.state.fail() + } + defer r.state.nextd(&err) + + var data []byte + if r.isNotConcurrent() { + size := r.frame.Descriptor.Flags.BlockSizeIndex() + data = size.Get() + defer lz4block.Put(data) + } + for { + var bn int + var dst []byte + if r.isNotConcurrent() { + bn, err = r.read(data) + dst = data[:bn] + } else { + lz4block.Put(dst) + dst = <-r.reads + bn = len(dst) + if bn == 0 { + // No uncompressed data: something went wrong or we are done. + err = r.frame.Blocks.ErrorR() + } + } + switch err { + case nil: + case io.EOF: + err = r.frame.CloseR(r.src) + return + default: + return + } + r.handler(bn) + bn, err = w.Write(dst) + n += int64(bn) + if err != nil { + return + } + } +} diff --git a/vendor/github.com/pierrec/lz4/v4/state.go b/vendor/github.com/pierrec/lz4/v4/state.go new file mode 100644 index 00000000..d94f04d0 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/state.go @@ -0,0 +1,75 @@ +package lz4 + +import ( + "errors" + "fmt" + "io" + + "github.com/pierrec/lz4/v4/internal/lz4errors" +) + +//go:generate go run golang.org/x/tools/cmd/stringer -type=aState -output state_gen.go + +const ( + noState aState = iota // uninitialized reader + errorState // unrecoverable error encountered + newState // instantiated object + readState // reading data + writeState // writing data + closedState // all done +) + +type ( + aState uint8 + _State struct { + states []aState + state aState + err error + } +) + +func (s *_State) init(states []aState) { + s.states = states + s.state = states[0] +} + +func (s *_State) reset() { + s.state = s.states[0] + s.err = nil +} + +// next sets the state to the next one unless it is passed a non nil error. +// It returns whether or not it is in error. +func (s *_State) next(err error) bool { + if err != nil { + s.err = fmt.Errorf("%s: %w", s.state, err) + s.state = errorState + return true + } + s.state = s.states[s.state] + return false +} + +// nextd is like next but for defers. +func (s *_State) nextd(errp *error) bool { + return errp != nil && s.next(*errp) +} + +// check sets s in error if not already in error and if the error is not nil or io.EOF, +func (s *_State) check(errp *error) { + if s.state == errorState || errp == nil { + return + } + if err := *errp; err != nil { + s.err = fmt.Errorf("%w[%s]", err, s.state) + if !errors.Is(err, io.EOF) { + s.state = errorState + } + } +} + +func (s *_State) fail() error { + s.state = errorState + s.err = fmt.Errorf("%w[%s]", lz4errors.ErrInternalUnhandledState, s.state) + return s.err +} diff --git a/vendor/github.com/pierrec/lz4/v4/state_gen.go b/vendor/github.com/pierrec/lz4/v4/state_gen.go new file mode 100644 index 00000000..75fb8289 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/state_gen.go @@ -0,0 +1,28 @@ +// Code generated by "stringer -type=aState -output state_gen.go"; DO NOT EDIT. + +package lz4 + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[noState-0] + _ = x[errorState-1] + _ = x[newState-2] + _ = x[readState-3] + _ = x[writeState-4] + _ = x[closedState-5] +} + +const _aState_name = "noStateerrorStatenewStatereadStatewriteStateclosedState" + +var _aState_index = [...]uint8{0, 7, 17, 25, 34, 44, 55} + +func (i aState) String() string { + if i >= aState(len(_aState_index)-1) { + return "aState(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _aState_name[_aState_index[i]:_aState_index[i+1]] +} diff --git a/vendor/github.com/pierrec/lz4/v4/writer.go b/vendor/github.com/pierrec/lz4/v4/writer.go new file mode 100644 index 00000000..44a43d25 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/writer.go @@ -0,0 +1,233 @@ +package lz4 + +import ( + "io" + + "github.com/pierrec/lz4/v4/internal/lz4block" + "github.com/pierrec/lz4/v4/internal/lz4errors" + "github.com/pierrec/lz4/v4/internal/lz4stream" +) + +var writerStates = []aState{ + noState: newState, + newState: writeState, + writeState: closedState, + closedState: newState, + errorState: newState, +} + +// NewWriter returns a new LZ4 frame encoder. +func NewWriter(w io.Writer) *Writer { + zw := &Writer{frame: lz4stream.NewFrame()} + zw.state.init(writerStates) + _ = zw.Apply(DefaultBlockSizeOption, DefaultChecksumOption, DefaultConcurrency, defaultOnBlockDone) + zw.Reset(w) + return zw +} + +// Writer allows writing an LZ4 stream. +type Writer struct { + state _State + src io.Writer // destination writer + level lz4block.CompressionLevel // how hard to try + num int // concurrency level + frame *lz4stream.Frame // frame being built + data []byte // pending data + idx int // size of pending data + handler func(int) + legacy bool +} + +func (*Writer) private() {} + +func (w *Writer) Apply(options ...Option) (err error) { + defer w.state.check(&err) + switch w.state.state { + case newState: + case errorState: + return w.state.err + default: + return lz4errors.ErrOptionClosedOrError + } + for _, o := range options { + if err = o(w); err != nil { + return + } + } + w.Reset(w.src) + return +} + +func (w *Writer) isNotConcurrent() bool { + return w.num == 1 +} + +// init sets up the Writer when in newState. It does not change the Writer state. +func (w *Writer) init() error { + w.frame.InitW(w.src, w.num, w.legacy) + if true || !w.isNotConcurrent() { + size := w.frame.Descriptor.Flags.BlockSizeIndex() + w.data = size.Get() + } + w.idx = 0 + return w.frame.Descriptor.Write(w.frame, w.src) +} + +func (w *Writer) Write(buf []byte) (n int, err error) { + defer w.state.check(&err) + switch w.state.state { + case writeState: + case closedState, errorState: + return 0, w.state.err + case newState: + if err = w.init(); w.state.next(err) { + return + } + default: + return 0, w.state.fail() + } + + zn := len(w.data) + for len(buf) > 0 { + if w.idx == 0 && len(buf) >= zn { + // Avoid a copy as there is enough data for a block. + if err = w.write(buf[:zn], false); err != nil { + return + } + n += zn + buf = buf[zn:] + continue + } + // Accumulate the data to be compressed. + m := copy(w.data[w.idx:], buf) + n += m + w.idx += m + buf = buf[m:] + + if w.idx < len(w.data) { + // Buffer not filled. + return + } + + // Buffer full. + if err = w.write(w.data, true); err != nil { + return + } + if !w.isNotConcurrent() { + size := w.frame.Descriptor.Flags.BlockSizeIndex() + w.data = size.Get() + } + w.idx = 0 + } + return +} + +func (w *Writer) write(data []byte, safe bool) error { + if w.isNotConcurrent() { + block := w.frame.Blocks.Block + err := block.Compress(w.frame, data, w.level).Write(w.frame, w.src) + w.handler(len(block.Data)) + return err + } + c := make(chan *lz4stream.FrameDataBlock) + w.frame.Blocks.Blocks <- c + go func(c chan *lz4stream.FrameDataBlock, data []byte, safe bool) { + b := lz4stream.NewFrameDataBlock(w.frame) + c <- b.Compress(w.frame, data, w.level) + <-c + w.handler(len(b.Data)) + b.Close(w.frame) + if safe { + // safe to put it back as the last usage of it was FrameDataBlock.Write() called before c is closed + lz4block.Put(data) + } + }(c, data, safe) + + return nil +} + +// Close closes the Writer, flushing any unwritten data to the underlying io.Writer, +// but does not close the underlying io.Writer. +func (w *Writer) Close() (err error) { + switch w.state.state { + case writeState: + case errorState: + return w.state.err + default: + return nil + } + defer w.state.nextd(&err) + if w.idx > 0 { + // Flush pending data, disable w.data freeing as it is done later on. + if err = w.write(w.data[:w.idx], false); err != nil { + return err + } + w.idx = 0 + } + err = w.frame.CloseW(w.src, w.num) + // It is now safe to free the buffer. + if w.data != nil { + lz4block.Put(w.data) + w.data = nil + } + return +} + +// Reset clears the state of the Writer w such that it is equivalent to its +// initial state from NewWriter, but instead writing to writer. +// Reset keeps the previous options unless overwritten by the supplied ones. +// No access to writer is performed. +// +// w.Close must be called before Reset or pending data may be dropped. +func (w *Writer) Reset(writer io.Writer) { + w.frame.Reset(w.num) + w.state.reset() + w.src = writer +} + +// ReadFrom efficiently reads from r and compressed into the Writer destination. +func (w *Writer) ReadFrom(r io.Reader) (n int64, err error) { + switch w.state.state { + case closedState, errorState: + return 0, w.state.err + case newState: + if err = w.init(); w.state.next(err) { + return + } + default: + return 0, w.state.fail() + } + defer w.state.check(&err) + + size := w.frame.Descriptor.Flags.BlockSizeIndex() + var done bool + var rn int + data := size.Get() + if w.isNotConcurrent() { + // Keep the same buffer for the whole process. + defer lz4block.Put(data) + } + for !done { + rn, err = io.ReadFull(r, data) + switch err { + case nil: + case io.EOF, io.ErrUnexpectedEOF: // read may be partial + done = true + default: + return + } + n += int64(rn) + err = w.write(data[:rn], true) + if err != nil { + return + } + w.handler(rn) + if !done && !w.isNotConcurrent() { + // The buffer will be returned automatically by go routines (safe=true) + // so get a new one fo the next round. + data = size.Get() + } + } + err = w.Close() + return +} diff --git a/vendor/github.com/pierrec/lz4/writer.go b/vendor/github.com/pierrec/lz4/writer.go deleted file mode 100644 index 01204380..00000000 --- a/vendor/github.com/pierrec/lz4/writer.go +++ /dev/null @@ -1,267 +0,0 @@ -package lz4 - -import ( - "encoding/binary" - "fmt" - "io" - - "github.com/pierrec/lz4/internal/xxh32" -) - -// Writer implements the LZ4 frame encoder. -type Writer struct { - Header - - buf [19]byte // magic number(4) + header(flags(2)+[Size(8)+DictID(4)]+checksum(1)) does not exceed 19 bytes - dst io.Writer // Destination. - checksum xxh32.XXHZero // Frame checksum. - zdata []byte // Compressed data. - data []byte // Data to be compressed. - idx int // Index into data. - hashtable [winSize]int // Hash table used in CompressBlock(). -} - -// NewWriter returns a new LZ4 frame encoder. -// No access to the underlying io.Writer is performed. -// The supplied Header is checked at the first Write. -// It is ok to change it before the first Write but then not until a Reset() is performed. -func NewWriter(dst io.Writer) *Writer { - return &Writer{dst: dst} -} - -// writeHeader builds and writes the header (magic+header) to the underlying io.Writer. -func (z *Writer) writeHeader() error { - // Default to 4Mb if BlockMaxSize is not set. - if z.Header.BlockMaxSize == 0 { - z.Header.BlockMaxSize = bsMapID[7] - } - // The only option that needs to be validated. - bSize := z.Header.BlockMaxSize - bSizeID, ok := bsMapValue[bSize] - if !ok { - return fmt.Errorf("lz4: invalid block max size: %d", bSize) - } - // Allocate the compressed/uncompressed buffers. - // The compressed buffer cannot exceed the uncompressed one. - if n := 2 * bSize; cap(z.zdata) < n { - z.zdata = make([]byte, n, n) - } - z.zdata = z.zdata[:bSize] - z.data = z.zdata[:cap(z.zdata)][bSize:] - z.idx = 0 - - // Size is optional. - buf := z.buf[:] - - // Set the fixed size data: magic number, block max size and flags. - binary.LittleEndian.PutUint32(buf[0:], frameMagic) - flg := byte(Version << 6) - flg |= 1 << 5 // No block dependency. - if z.Header.BlockChecksum { - flg |= 1 << 4 - } - if z.Header.Size > 0 { - flg |= 1 << 3 - } - if !z.Header.NoChecksum { - flg |= 1 << 2 - } - buf[4] = flg - buf[5] = bSizeID << 4 - - // Current buffer size: magic(4) + flags(1) + block max size (1). - n := 6 - // Optional items. - if z.Header.Size > 0 { - binary.LittleEndian.PutUint64(buf[n:], z.Header.Size) - n += 8 - } - - // The header checksum includes the flags, block max size and optional Size. - buf[n] = byte(xxh32.ChecksumZero(buf[4:n]) >> 8 & 0xFF) - z.checksum.Reset() - - // Header ready, write it out. - if _, err := z.dst.Write(buf[0 : n+1]); err != nil { - return err - } - z.Header.done = true - if debugFlag { - debug("wrote header %v", z.Header) - } - - return nil -} - -// Write compresses data from the supplied buffer into the underlying io.Writer. -// Write does not return until the data has been written. -func (z *Writer) Write(buf []byte) (int, error) { - if !z.Header.done { - if err := z.writeHeader(); err != nil { - return 0, err - } - } - if debugFlag { - debug("input buffer len=%d index=%d", len(buf), z.idx) - } - - zn := len(z.data) - var n int - for len(buf) > 0 { - if z.idx == 0 && len(buf) >= zn { - // Avoid a copy as there is enough data for a block. - if err := z.compressBlock(buf[:zn]); err != nil { - return n, err - } - n += zn - buf = buf[zn:] - continue - } - // Accumulate the data to be compressed. - m := copy(z.data[z.idx:], buf) - n += m - z.idx += m - buf = buf[m:] - if debugFlag { - debug("%d bytes copied to buf, current index %d", n, z.idx) - } - - if z.idx < len(z.data) { - // Buffer not filled. - if debugFlag { - debug("need more data for compression") - } - return n, nil - } - - // Buffer full. - if err := z.compressBlock(z.data); err != nil { - return n, err - } - z.idx = 0 - } - - return n, nil -} - -// compressBlock compresses a block. -func (z *Writer) compressBlock(data []byte) error { - if !z.NoChecksum { - z.checksum.Write(data) - } - - // The compressed block size cannot exceed the input's. - var zn int - var err error - - if level := z.Header.CompressionLevel; level != 0 { - zn, err = CompressBlockHC(data, z.zdata, level) - } else { - zn, err = CompressBlock(data, z.zdata, z.hashtable[:]) - } - - var zdata []byte - var bLen uint32 - if debugFlag { - debug("block compression %d => %d", len(data), zn) - } - if err == nil && zn > 0 && zn < len(data) { - // Compressible and compressed size smaller than uncompressed: ok! - bLen = uint32(zn) - zdata = z.zdata[:zn] - } else { - // Uncompressed block. - bLen = uint32(len(data)) | compressedBlockFlag - zdata = data - } - if debugFlag { - debug("block compression to be written len=%d data len=%d", bLen, len(zdata)) - } - - // Write the block. - if err := z.writeUint32(bLen); err != nil { - return err - } - if _, err := z.dst.Write(zdata); err != nil { - return err - } - - if z.BlockChecksum { - checksum := xxh32.ChecksumZero(zdata) - if debugFlag { - debug("block checksum %x", checksum) - } - if err := z.writeUint32(checksum); err != nil { - return err - } - } - if debugFlag { - debug("current frame checksum %x", z.checksum.Sum32()) - } - - return nil -} - -// Flush flushes any pending compressed data to the underlying writer. -// Flush does not return until the data has been written. -// If the underlying writer returns an error, Flush returns that error. -func (z *Writer) Flush() error { - if debugFlag { - debug("flush with index %d", z.idx) - } - if z.idx == 0 { - return nil - } - - return z.compressBlock(z.data[:z.idx]) -} - -// Close closes the Writer, flushing any unwritten data to the underlying io.Writer, but does not close the underlying io.Writer. -func (z *Writer) Close() error { - if !z.Header.done { - if err := z.writeHeader(); err != nil { - return err - } - } - - if err := z.Flush(); err != nil { - return err - } - - if debugFlag { - debug("writing last empty block") - } - if err := z.writeUint32(0); err != nil { - return err - } - if !z.NoChecksum { - checksum := z.checksum.Sum32() - if debugFlag { - debug("stream checksum %x", checksum) - } - if err := z.writeUint32(checksum); err != nil { - return err - } - } - return nil -} - -// Reset clears the state of the Writer z such that it is equivalent to its -// initial state from NewWriter, but instead writing to w. -// No access to the underlying io.Writer is performed. -func (z *Writer) Reset(w io.Writer) { - z.Header = Header{} - z.dst = w - z.checksum.Reset() - z.zdata = z.zdata[:0] - z.data = z.data[:0] - z.idx = 0 -} - -// writeUint32 writes a uint32 to the underlying writer. -func (z *Writer) writeUint32(x uint32) error { - buf := z.buf[:4] - binary.LittleEndian.PutUint32(buf, x) - _, err := z.dst.Write(buf) - return err -} diff --git a/vendor/golang.org/x/mod/modfile/print.go b/vendor/golang.org/x/mod/modfile/print.go new file mode 100644 index 00000000..2a0123d4 --- /dev/null +++ b/vendor/golang.org/x/mod/modfile/print.go @@ -0,0 +1,184 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Module file printer. + +package modfile + +import ( + "bytes" + "fmt" + "strings" +) + +// Format returns a go.mod file as a byte slice, formatted in standard style. +func Format(f *FileSyntax) []byte { + pr := &printer{} + pr.file(f) + + // remove trailing blank lines + b := pr.Bytes() + for len(b) > 0 && b[len(b)-1] == '\n' && (len(b) == 1 || b[len(b)-2] == '\n') { + b = b[:len(b)-1] + } + return b +} + +// A printer collects the state during printing of a file or expression. +type printer struct { + bytes.Buffer // output buffer + comment []Comment // pending end-of-line comments + margin int // left margin (indent), a number of tabs +} + +// printf prints to the buffer. +func (p *printer) printf(format string, args ...interface{}) { + fmt.Fprintf(p, format, args...) +} + +// indent returns the position on the current line, in bytes, 0-indexed. +func (p *printer) indent() int { + b := p.Bytes() + n := 0 + for n < len(b) && b[len(b)-1-n] != '\n' { + n++ + } + return n +} + +// newline ends the current line, flushing end-of-line comments. +func (p *printer) newline() { + if len(p.comment) > 0 { + p.printf(" ") + for i, com := range p.comment { + if i > 0 { + p.trim() + p.printf("\n") + for i := 0; i < p.margin; i++ { + p.printf("\t") + } + } + p.printf("%s", strings.TrimSpace(com.Token)) + } + p.comment = p.comment[:0] + } + + p.trim() + if b := p.Bytes(); len(b) == 0 || (len(b) >= 2 && b[len(b)-1] == '\n' && b[len(b)-2] == '\n') { + // skip the blank line at top of file or after a blank line + } else { + p.printf("\n") + } + for i := 0; i < p.margin; i++ { + p.printf("\t") + } +} + +// trim removes trailing spaces and tabs from the current line. +func (p *printer) trim() { + // Remove trailing spaces and tabs from line we're about to end. + b := p.Bytes() + n := len(b) + for n > 0 && (b[n-1] == '\t' || b[n-1] == ' ') { + n-- + } + p.Truncate(n) +} + +// file formats the given file into the print buffer. +func (p *printer) file(f *FileSyntax) { + for _, com := range f.Before { + p.printf("%s", strings.TrimSpace(com.Token)) + p.newline() + } + + for i, stmt := range f.Stmt { + switch x := stmt.(type) { + case *CommentBlock: + // comments already handled + p.expr(x) + + default: + p.expr(x) + p.newline() + } + + for _, com := range stmt.Comment().After { + p.printf("%s", strings.TrimSpace(com.Token)) + p.newline() + } + + if i+1 < len(f.Stmt) { + p.newline() + } + } +} + +func (p *printer) expr(x Expr) { + // Emit line-comments preceding this expression. + if before := x.Comment().Before; len(before) > 0 { + // Want to print a line comment. + // Line comments must be at the current margin. + p.trim() + if p.indent() > 0 { + // There's other text on the line. Start a new line. + p.printf("\n") + } + // Re-indent to margin. + for i := 0; i < p.margin; i++ { + p.printf("\t") + } + for _, com := range before { + p.printf("%s", strings.TrimSpace(com.Token)) + p.newline() + } + } + + switch x := x.(type) { + default: + panic(fmt.Errorf("printer: unexpected type %T", x)) + + case *CommentBlock: + // done + + case *LParen: + p.printf("(") + case *RParen: + p.printf(")") + + case *Line: + p.tokens(x.Token) + + case *LineBlock: + p.tokens(x.Token) + p.printf(" ") + p.expr(&x.LParen) + p.margin++ + for _, l := range x.Line { + p.newline() + p.expr(l) + } + p.margin-- + p.newline() + p.expr(&x.RParen) + } + + // Queue end-of-line comments for printing when we + // reach the end of the line. + p.comment = append(p.comment, x.Comment().Suffix...) +} + +func (p *printer) tokens(tokens []string) { + sep := "" + for _, t := range tokens { + if t == "," || t == ")" || t == "]" || t == "}" { + sep = "" + } + p.printf("%s%s", sep, t) + sep = " " + if t == "(" || t == "[" || t == "{" { + sep = "" + } + } +} diff --git a/vendor/golang.org/x/mod/modfile/read.go b/vendor/golang.org/x/mod/modfile/read.go new file mode 100644 index 00000000..de1b9821 --- /dev/null +++ b/vendor/golang.org/x/mod/modfile/read.go @@ -0,0 +1,959 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modfile + +import ( + "bytes" + "errors" + "fmt" + "os" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +// A Position describes an arbitrary source position in a file, including the +// file, line, column, and byte offset. +type Position struct { + Line int // line in input (starting at 1) + LineRune int // rune in line (starting at 1) + Byte int // byte in input (starting at 0) +} + +// add returns the position at the end of s, assuming it starts at p. +func (p Position) add(s string) Position { + p.Byte += len(s) + if n := strings.Count(s, "\n"); n > 0 { + p.Line += n + s = s[strings.LastIndex(s, "\n")+1:] + p.LineRune = 1 + } + p.LineRune += utf8.RuneCountInString(s) + return p +} + +// An Expr represents an input element. +type Expr interface { + // Span returns the start and end position of the expression, + // excluding leading or trailing comments. + Span() (start, end Position) + + // Comment returns the comments attached to the expression. + // This method would normally be named 'Comments' but that + // would interfere with embedding a type of the same name. + Comment() *Comments +} + +// A Comment represents a single // comment. +type Comment struct { + Start Position + Token string // without trailing newline + Suffix bool // an end of line (not whole line) comment +} + +// Comments collects the comments associated with an expression. +type Comments struct { + Before []Comment // whole-line comments before this expression + Suffix []Comment // end-of-line comments after this expression + + // For top-level expressions only, After lists whole-line + // comments following the expression. + After []Comment +} + +// Comment returns the receiver. This isn't useful by itself, but +// a [Comments] struct is embedded into all the expression +// implementation types, and this gives each of those a Comment +// method to satisfy the Expr interface. +func (c *Comments) Comment() *Comments { + return c +} + +// A FileSyntax represents an entire go.mod file. +type FileSyntax struct { + Name string // file path + Comments + Stmt []Expr +} + +func (x *FileSyntax) Span() (start, end Position) { + if len(x.Stmt) == 0 { + return + } + start, _ = x.Stmt[0].Span() + _, end = x.Stmt[len(x.Stmt)-1].Span() + return start, end +} + +// addLine adds a line containing the given tokens to the file. +// +// If the first token of the hint matches the first token of the +// line, the new line is added at the end of the block containing hint, +// extracting hint into a new block if it is not yet in one. +// +// If the hint is non-nil buts its first token does not match, +// the new line is added after the block containing hint +// (or hint itself, if not in a block). +// +// If no hint is provided, addLine appends the line to the end of +// the last block with a matching first token, +// or to the end of the file if no such block exists. +func (x *FileSyntax) addLine(hint Expr, tokens ...string) *Line { + if hint == nil { + // If no hint given, add to the last statement of the given type. + Loop: + for i := len(x.Stmt) - 1; i >= 0; i-- { + stmt := x.Stmt[i] + switch stmt := stmt.(type) { + case *Line: + if stmt.Token != nil && stmt.Token[0] == tokens[0] { + hint = stmt + break Loop + } + case *LineBlock: + if stmt.Token[0] == tokens[0] { + hint = stmt + break Loop + } + } + } + } + + newLineAfter := func(i int) *Line { + new := &Line{Token: tokens} + if i == len(x.Stmt) { + x.Stmt = append(x.Stmt, new) + } else { + x.Stmt = append(x.Stmt, nil) + copy(x.Stmt[i+2:], x.Stmt[i+1:]) + x.Stmt[i+1] = new + } + return new + } + + if hint != nil { + for i, stmt := range x.Stmt { + switch stmt := stmt.(type) { + case *Line: + if stmt == hint { + if stmt.Token == nil || stmt.Token[0] != tokens[0] { + return newLineAfter(i) + } + + // Convert line to line block. + stmt.InBlock = true + block := &LineBlock{Token: stmt.Token[:1], Line: []*Line{stmt}} + stmt.Token = stmt.Token[1:] + x.Stmt[i] = block + new := &Line{Token: tokens[1:], InBlock: true} + block.Line = append(block.Line, new) + return new + } + + case *LineBlock: + if stmt == hint { + if stmt.Token[0] != tokens[0] { + return newLineAfter(i) + } + + new := &Line{Token: tokens[1:], InBlock: true} + stmt.Line = append(stmt.Line, new) + return new + } + + for j, line := range stmt.Line { + if line == hint { + if stmt.Token[0] != tokens[0] { + return newLineAfter(i) + } + + // Add new line after hint within the block. + stmt.Line = append(stmt.Line, nil) + copy(stmt.Line[j+2:], stmt.Line[j+1:]) + new := &Line{Token: tokens[1:], InBlock: true} + stmt.Line[j+1] = new + return new + } + } + } + } + } + + new := &Line{Token: tokens} + x.Stmt = append(x.Stmt, new) + return new +} + +func (x *FileSyntax) updateLine(line *Line, tokens ...string) { + if line.InBlock { + tokens = tokens[1:] + } + line.Token = tokens +} + +// markRemoved modifies line so that it (and its end-of-line comment, if any) +// will be dropped by (*FileSyntax).Cleanup. +func (line *Line) markRemoved() { + line.Token = nil + line.Comments.Suffix = nil +} + +// Cleanup cleans up the file syntax x after any edit operations. +// To avoid quadratic behavior, (*Line).markRemoved marks the line as dead +// by setting line.Token = nil but does not remove it from the slice +// in which it appears. After edits have all been indicated, +// calling Cleanup cleans out the dead lines. +func (x *FileSyntax) Cleanup() { + w := 0 + for _, stmt := range x.Stmt { + switch stmt := stmt.(type) { + case *Line: + if stmt.Token == nil { + continue + } + case *LineBlock: + ww := 0 + for _, line := range stmt.Line { + if line.Token != nil { + stmt.Line[ww] = line + ww++ + } + } + if ww == 0 { + continue + } + if ww == 1 && len(stmt.RParen.Comments.Before) == 0 { + // Collapse block into single line but keep the Line reference used by the + // parsed File structure. + *stmt.Line[0] = Line{ + Comments: Comments{ + Before: commentsAdd(stmt.Before, stmt.Line[0].Before), + Suffix: commentsAdd(stmt.Line[0].Suffix, stmt.Suffix), + After: commentsAdd(stmt.Line[0].After, stmt.After), + }, + Token: stringsAdd(stmt.Token, stmt.Line[0].Token), + } + x.Stmt[w] = stmt.Line[0] + w++ + continue + } + stmt.Line = stmt.Line[:ww] + } + x.Stmt[w] = stmt + w++ + } + x.Stmt = x.Stmt[:w] +} + +func commentsAdd(x, y []Comment) []Comment { + return append(x[:len(x):len(x)], y...) +} + +func stringsAdd(x, y []string) []string { + return append(x[:len(x):len(x)], y...) +} + +// A CommentBlock represents a top-level block of comments separate +// from any rule. +type CommentBlock struct { + Comments + Start Position +} + +func (x *CommentBlock) Span() (start, end Position) { + return x.Start, x.Start +} + +// A Line is a single line of tokens. +type Line struct { + Comments + Start Position + Token []string + InBlock bool + End Position +} + +func (x *Line) Span() (start, end Position) { + return x.Start, x.End +} + +// A LineBlock is a factored block of lines, like +// +// require ( +// "x" +// "y" +// ) +type LineBlock struct { + Comments + Start Position + LParen LParen + Token []string + Line []*Line + RParen RParen +} + +func (x *LineBlock) Span() (start, end Position) { + return x.Start, x.RParen.Pos.add(")") +} + +// An LParen represents the beginning of a parenthesized line block. +// It is a place to store suffix comments. +type LParen struct { + Comments + Pos Position +} + +func (x *LParen) Span() (start, end Position) { + return x.Pos, x.Pos.add(")") +} + +// An RParen represents the end of a parenthesized line block. +// It is a place to store whole-line (before) comments. +type RParen struct { + Comments + Pos Position +} + +func (x *RParen) Span() (start, end Position) { + return x.Pos, x.Pos.add(")") +} + +// An input represents a single input file being parsed. +type input struct { + // Lexing state. + filename string // name of input file, for errors + complete []byte // entire input + remaining []byte // remaining input + tokenStart []byte // token being scanned to end of input + token token // next token to be returned by lex, peek + pos Position // current input position + comments []Comment // accumulated comments + + // Parser state. + file *FileSyntax // returned top-level syntax tree + parseErrors ErrorList // errors encountered during parsing + + // Comment assignment state. + pre []Expr // all expressions, in preorder traversal + post []Expr // all expressions, in postorder traversal +} + +func newInput(filename string, data []byte) *input { + return &input{ + filename: filename, + complete: data, + remaining: data, + pos: Position{Line: 1, LineRune: 1, Byte: 0}, + } +} + +// parse parses the input file. +func parse(file string, data []byte) (f *FileSyntax, err error) { + // The parser panics for both routine errors like syntax errors + // and for programmer bugs like array index errors. + // Turn both into error returns. Catching bug panics is + // especially important when processing many files. + in := newInput(file, data) + defer func() { + if e := recover(); e != nil && e != &in.parseErrors { + in.parseErrors = append(in.parseErrors, Error{ + Filename: in.filename, + Pos: in.pos, + Err: fmt.Errorf("internal error: %v", e), + }) + } + if err == nil && len(in.parseErrors) > 0 { + err = in.parseErrors + } + }() + + // Prime the lexer by reading in the first token. It will be available + // in the next peek() or lex() call. + in.readToken() + + // Invoke the parser. + in.parseFile() + if len(in.parseErrors) > 0 { + return nil, in.parseErrors + } + in.file.Name = in.filename + + // Assign comments to nearby syntax. + in.assignComments() + + return in.file, nil +} + +// Error is called to report an error. +// Error does not return: it panics. +func (in *input) Error(s string) { + in.parseErrors = append(in.parseErrors, Error{ + Filename: in.filename, + Pos: in.pos, + Err: errors.New(s), + }) + panic(&in.parseErrors) +} + +// eof reports whether the input has reached end of file. +func (in *input) eof() bool { + return len(in.remaining) == 0 +} + +// peekRune returns the next rune in the input without consuming it. +func (in *input) peekRune() int { + if len(in.remaining) == 0 { + return 0 + } + r, _ := utf8.DecodeRune(in.remaining) + return int(r) +} + +// peekPrefix reports whether the remaining input begins with the given prefix. +func (in *input) peekPrefix(prefix string) bool { + // This is like bytes.HasPrefix(in.remaining, []byte(prefix)) + // but without the allocation of the []byte copy of prefix. + for i := 0; i < len(prefix); i++ { + if i >= len(in.remaining) || in.remaining[i] != prefix[i] { + return false + } + } + return true +} + +// readRune consumes and returns the next rune in the input. +func (in *input) readRune() int { + if len(in.remaining) == 0 { + in.Error("internal lexer error: readRune at EOF") + } + r, size := utf8.DecodeRune(in.remaining) + in.remaining = in.remaining[size:] + if r == '\n' { + in.pos.Line++ + in.pos.LineRune = 1 + } else { + in.pos.LineRune++ + } + in.pos.Byte += size + return int(r) +} + +type token struct { + kind tokenKind + pos Position + endPos Position + text string +} + +type tokenKind int + +const ( + _EOF tokenKind = -(iota + 1) + _EOLCOMMENT + _IDENT + _STRING + _COMMENT + + // newlines and punctuation tokens are allowed as ASCII codes. +) + +func (k tokenKind) isComment() bool { + return k == _COMMENT || k == _EOLCOMMENT +} + +// isEOL returns whether a token terminates a line. +func (k tokenKind) isEOL() bool { + return k == _EOF || k == _EOLCOMMENT || k == '\n' +} + +// startToken marks the beginning of the next input token. +// It must be followed by a call to endToken, once the token's text has +// been consumed using readRune. +func (in *input) startToken() { + in.tokenStart = in.remaining + in.token.text = "" + in.token.pos = in.pos +} + +// endToken marks the end of an input token. +// It records the actual token string in tok.text. +// A single trailing newline (LF or CRLF) will be removed from comment tokens. +func (in *input) endToken(kind tokenKind) { + in.token.kind = kind + text := string(in.tokenStart[:len(in.tokenStart)-len(in.remaining)]) + if kind.isComment() { + if strings.HasSuffix(text, "\r\n") { + text = text[:len(text)-2] + } else { + text = strings.TrimSuffix(text, "\n") + } + } + in.token.text = text + in.token.endPos = in.pos +} + +// peek returns the kind of the next token returned by lex. +func (in *input) peek() tokenKind { + return in.token.kind +} + +// lex is called from the parser to obtain the next input token. +func (in *input) lex() token { + tok := in.token + in.readToken() + return tok +} + +// readToken lexes the next token from the text and stores it in in.token. +func (in *input) readToken() { + // Skip past spaces, stopping at non-space or EOF. + for !in.eof() { + c := in.peekRune() + if c == ' ' || c == '\t' || c == '\r' { + in.readRune() + continue + } + + // Comment runs to end of line. + if in.peekPrefix("//") { + in.startToken() + + // Is this comment the only thing on its line? + // Find the last \n before this // and see if it's all + // spaces from there to here. + i := bytes.LastIndex(in.complete[:in.pos.Byte], []byte("\n")) + suffix := len(bytes.TrimSpace(in.complete[i+1:in.pos.Byte])) > 0 + in.readRune() + in.readRune() + + // Consume comment. + for len(in.remaining) > 0 && in.readRune() != '\n' { + } + + // If we are at top level (not in a statement), hand the comment to + // the parser as a _COMMENT token. The grammar is written + // to handle top-level comments itself. + if !suffix { + in.endToken(_COMMENT) + return + } + + // Otherwise, save comment for later attachment to syntax tree. + in.endToken(_EOLCOMMENT) + in.comments = append(in.comments, Comment{in.token.pos, in.token.text, suffix}) + return + } + + if in.peekPrefix("/*") { + in.Error("mod files must use // comments (not /* */ comments)") + } + + // Found non-space non-comment. + break + } + + // Found the beginning of the next token. + in.startToken() + + // End of file. + if in.eof() { + in.endToken(_EOF) + return + } + + // Punctuation tokens. + switch c := in.peekRune(); c { + case '\n', '(', ')', '[', ']', '{', '}', ',': + in.readRune() + in.endToken(tokenKind(c)) + return + + case '"', '`': // quoted string + quote := c + in.readRune() + for { + if in.eof() { + in.pos = in.token.pos + in.Error("unexpected EOF in string") + } + if in.peekRune() == '\n' { + in.Error("unexpected newline in string") + } + c := in.readRune() + if c == quote { + break + } + if c == '\\' && quote != '`' { + if in.eof() { + in.pos = in.token.pos + in.Error("unexpected EOF in string") + } + in.readRune() + } + } + in.endToken(_STRING) + return + } + + // Checked all punctuation. Must be identifier token. + if c := in.peekRune(); !isIdent(c) { + in.Error(fmt.Sprintf("unexpected input character %#q", c)) + } + + // Scan over identifier. + for isIdent(in.peekRune()) { + if in.peekPrefix("//") { + break + } + if in.peekPrefix("/*") { + in.Error("mod files must use // comments (not /* */ comments)") + } + in.readRune() + } + in.endToken(_IDENT) +} + +// isIdent reports whether c is an identifier rune. +// We treat most printable runes as identifier runes, except for a handful of +// ASCII punctuation characters. +func isIdent(c int) bool { + switch r := rune(c); r { + case ' ', '(', ')', '[', ']', '{', '}', ',': + return false + default: + return !unicode.IsSpace(r) && unicode.IsPrint(r) + } +} + +// Comment assignment. +// We build two lists of all subexpressions, preorder and postorder. +// The preorder list is ordered by start location, with outer expressions first. +// The postorder list is ordered by end location, with outer expressions last. +// We use the preorder list to assign each whole-line comment to the syntax +// immediately following it, and we use the postorder list to assign each +// end-of-line comment to the syntax immediately preceding it. + +// order walks the expression adding it and its subexpressions to the +// preorder and postorder lists. +func (in *input) order(x Expr) { + if x != nil { + in.pre = append(in.pre, x) + } + switch x := x.(type) { + default: + panic(fmt.Errorf("order: unexpected type %T", x)) + case nil: + // nothing + case *LParen, *RParen: + // nothing + case *CommentBlock: + // nothing + case *Line: + // nothing + case *FileSyntax: + for _, stmt := range x.Stmt { + in.order(stmt) + } + case *LineBlock: + in.order(&x.LParen) + for _, l := range x.Line { + in.order(l) + } + in.order(&x.RParen) + } + if x != nil { + in.post = append(in.post, x) + } +} + +// assignComments attaches comments to nearby syntax. +func (in *input) assignComments() { + const debug = false + + // Generate preorder and postorder lists. + in.order(in.file) + + // Split into whole-line comments and suffix comments. + var line, suffix []Comment + for _, com := range in.comments { + if com.Suffix { + suffix = append(suffix, com) + } else { + line = append(line, com) + } + } + + if debug { + for _, c := range line { + fmt.Fprintf(os.Stderr, "LINE %q :%d:%d #%d\n", c.Token, c.Start.Line, c.Start.LineRune, c.Start.Byte) + } + } + + // Assign line comments to syntax immediately following. + for _, x := range in.pre { + start, _ := x.Span() + if debug { + fmt.Fprintf(os.Stderr, "pre %T :%d:%d #%d\n", x, start.Line, start.LineRune, start.Byte) + } + xcom := x.Comment() + for len(line) > 0 && start.Byte >= line[0].Start.Byte { + if debug { + fmt.Fprintf(os.Stderr, "ASSIGN LINE %q #%d\n", line[0].Token, line[0].Start.Byte) + } + xcom.Before = append(xcom.Before, line[0]) + line = line[1:] + } + } + + // Remaining line comments go at end of file. + in.file.After = append(in.file.After, line...) + + if debug { + for _, c := range suffix { + fmt.Fprintf(os.Stderr, "SUFFIX %q :%d:%d #%d\n", c.Token, c.Start.Line, c.Start.LineRune, c.Start.Byte) + } + } + + // Assign suffix comments to syntax immediately before. + for i := len(in.post) - 1; i >= 0; i-- { + x := in.post[i] + + start, end := x.Span() + if debug { + fmt.Fprintf(os.Stderr, "post %T :%d:%d #%d :%d:%d #%d\n", x, start.Line, start.LineRune, start.Byte, end.Line, end.LineRune, end.Byte) + } + + // Do not assign suffix comments to end of line block or whole file. + // Instead assign them to the last element inside. + switch x.(type) { + case *FileSyntax: + continue + } + + // Do not assign suffix comments to something that starts + // on an earlier line, so that in + // + // x ( y + // z ) // comment + // + // we assign the comment to z and not to x ( ... ). + if start.Line != end.Line { + continue + } + xcom := x.Comment() + for len(suffix) > 0 && end.Byte <= suffix[len(suffix)-1].Start.Byte { + if debug { + fmt.Fprintf(os.Stderr, "ASSIGN SUFFIX %q #%d\n", suffix[len(suffix)-1].Token, suffix[len(suffix)-1].Start.Byte) + } + xcom.Suffix = append(xcom.Suffix, suffix[len(suffix)-1]) + suffix = suffix[:len(suffix)-1] + } + } + + // We assigned suffix comments in reverse. + // If multiple suffix comments were appended to the same + // expression node, they are now in reverse. Fix that. + for _, x := range in.post { + reverseComments(x.Comment().Suffix) + } + + // Remaining suffix comments go at beginning of file. + in.file.Before = append(in.file.Before, suffix...) +} + +// reverseComments reverses the []Comment list. +func reverseComments(list []Comment) { + for i, j := 0, len(list)-1; i < j; i, j = i+1, j-1 { + list[i], list[j] = list[j], list[i] + } +} + +func (in *input) parseFile() { + in.file = new(FileSyntax) + var cb *CommentBlock + for { + switch in.peek() { + case '\n': + in.lex() + if cb != nil { + in.file.Stmt = append(in.file.Stmt, cb) + cb = nil + } + case _COMMENT: + tok := in.lex() + if cb == nil { + cb = &CommentBlock{Start: tok.pos} + } + com := cb.Comment() + com.Before = append(com.Before, Comment{Start: tok.pos, Token: tok.text}) + case _EOF: + if cb != nil { + in.file.Stmt = append(in.file.Stmt, cb) + } + return + default: + in.parseStmt() + if cb != nil { + in.file.Stmt[len(in.file.Stmt)-1].Comment().Before = cb.Before + cb = nil + } + } + } +} + +func (in *input) parseStmt() { + tok := in.lex() + start := tok.pos + end := tok.endPos + tokens := []string{tok.text} + for { + tok := in.lex() + switch { + case tok.kind.isEOL(): + in.file.Stmt = append(in.file.Stmt, &Line{ + Start: start, + Token: tokens, + End: end, + }) + return + + case tok.kind == '(': + if next := in.peek(); next.isEOL() { + // Start of block: no more tokens on this line. + in.file.Stmt = append(in.file.Stmt, in.parseLineBlock(start, tokens, tok)) + return + } else if next == ')' { + rparen := in.lex() + if in.peek().isEOL() { + // Empty block. + in.lex() + in.file.Stmt = append(in.file.Stmt, &LineBlock{ + Start: start, + Token: tokens, + LParen: LParen{Pos: tok.pos}, + RParen: RParen{Pos: rparen.pos}, + }) + return + } + // '( )' in the middle of the line, not a block. + tokens = append(tokens, tok.text, rparen.text) + } else { + // '(' in the middle of the line, not a block. + tokens = append(tokens, tok.text) + } + + default: + tokens = append(tokens, tok.text) + end = tok.endPos + } + } +} + +func (in *input) parseLineBlock(start Position, token []string, lparen token) *LineBlock { + x := &LineBlock{ + Start: start, + Token: token, + LParen: LParen{Pos: lparen.pos}, + } + var comments []Comment + for { + switch in.peek() { + case _EOLCOMMENT: + // Suffix comment, will be attached later by assignComments. + in.lex() + case '\n': + // Blank line. Add an empty comment to preserve it. + in.lex() + if len(comments) == 0 && len(x.Line) > 0 || len(comments) > 0 && comments[len(comments)-1].Token != "" { + comments = append(comments, Comment{}) + } + case _COMMENT: + tok := in.lex() + comments = append(comments, Comment{Start: tok.pos, Token: tok.text}) + case _EOF: + in.Error(fmt.Sprintf("syntax error (unterminated block started at %s:%d:%d)", in.filename, x.Start.Line, x.Start.LineRune)) + case ')': + rparen := in.lex() + x.RParen.Before = comments + x.RParen.Pos = rparen.pos + if !in.peek().isEOL() { + in.Error("syntax error (expected newline after closing paren)") + } + in.lex() + return x + default: + l := in.parseLine() + x.Line = append(x.Line, l) + l.Comment().Before = comments + comments = nil + } + } +} + +func (in *input) parseLine() *Line { + tok := in.lex() + if tok.kind.isEOL() { + in.Error("internal parse error: parseLine at end of line") + } + start := tok.pos + end := tok.endPos + tokens := []string{tok.text} + for { + tok := in.lex() + if tok.kind.isEOL() { + return &Line{ + Start: start, + Token: tokens, + End: end, + InBlock: true, + } + } + tokens = append(tokens, tok.text) + end = tok.endPos + } +} + +var ( + slashSlash = []byte("//") + moduleStr = []byte("module") +) + +// ModulePath returns the module path from the gomod file text. +// If it cannot find a module path, it returns an empty string. +// It is tolerant of unrelated problems in the go.mod file. +func ModulePath(mod []byte) string { + for len(mod) > 0 { + line := mod + mod = nil + if i := bytes.IndexByte(line, '\n'); i >= 0 { + line, mod = line[:i], line[i+1:] + } + if i := bytes.Index(line, slashSlash); i >= 0 { + line = line[:i] + } + line = bytes.TrimSpace(line) + if !bytes.HasPrefix(line, moduleStr) { + continue + } + line = line[len(moduleStr):] + n := len(line) + line = bytes.TrimSpace(line) + if len(line) == n || len(line) == 0 { + continue + } + + if line[0] == '"' || line[0] == '`' { + p, err := strconv.Unquote(string(line)) + if err != nil { + return "" // malformed quoted string or multiline module path + } + return p + } + + return string(line) + } + return "" // missing module path +} diff --git a/vendor/golang.org/x/mod/modfile/rule.go b/vendor/golang.org/x/mod/modfile/rule.go new file mode 100644 index 00000000..3e4a1d0a --- /dev/null +++ b/vendor/golang.org/x/mod/modfile/rule.go @@ -0,0 +1,1836 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package modfile implements a parser and formatter for go.mod files. +// +// The go.mod syntax is described in +// https://pkg.go.dev/cmd/go/#hdr-The_go_mod_file. +// +// The [Parse] and [ParseLax] functions both parse a go.mod file and return an +// abstract syntax tree. ParseLax ignores unknown statements and may be used to +// parse go.mod files that may have been developed with newer versions of Go. +// +// The [File] struct returned by Parse and ParseLax represent an abstract +// go.mod file. File has several methods like [File.AddNewRequire] and +// [File.DropReplace] that can be used to programmatically edit a file. +// +// The [Format] function formats a File back to a byte slice which can be +// written to a file. +package modfile + +import ( + "errors" + "fmt" + "path/filepath" + "sort" + "strconv" + "strings" + "unicode" + + "golang.org/x/mod/internal/lazyregexp" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +// A File is the parsed, interpreted form of a go.mod file. +type File struct { + Module *Module + Go *Go + Toolchain *Toolchain + Godebug []*Godebug + Require []*Require + Exclude []*Exclude + Replace []*Replace + Retract []*Retract + Tool []*Tool + + Syntax *FileSyntax +} + +// A Module is the module statement. +type Module struct { + Mod module.Version + Deprecated string + Syntax *Line +} + +// A Go is the go statement. +type Go struct { + Version string // "1.23" + Syntax *Line +} + +// A Toolchain is the toolchain statement. +type Toolchain struct { + Name string // "go1.21rc1" + Syntax *Line +} + +// A Godebug is a single godebug key=value statement. +type Godebug struct { + Key string + Value string + Syntax *Line +} + +// An Exclude is a single exclude statement. +type Exclude struct { + Mod module.Version + Syntax *Line +} + +// A Replace is a single replace statement. +type Replace struct { + Old module.Version + New module.Version + Syntax *Line +} + +// A Retract is a single retract statement. +type Retract struct { + VersionInterval + Rationale string + Syntax *Line +} + +// A Tool is a single tool statement. +type Tool struct { + Path string + Syntax *Line +} + +// A VersionInterval represents a range of versions with upper and lower bounds. +// Intervals are closed: both bounds are included. When Low is equal to High, +// the interval may refer to a single version ('v1.2.3') or an interval +// ('[v1.2.3, v1.2.3]'); both have the same representation. +type VersionInterval struct { + Low, High string +} + +// A Require is a single require statement. +type Require struct { + Mod module.Version + Indirect bool // has "// indirect" comment + Syntax *Line +} + +func (r *Require) markRemoved() { + r.Syntax.markRemoved() + *r = Require{} +} + +func (r *Require) setVersion(v string) { + r.Mod.Version = v + + if line := r.Syntax; len(line.Token) > 0 { + if line.InBlock { + // If the line is preceded by an empty line, remove it; see + // https://golang.org/issue/33779. + if len(line.Comments.Before) == 1 && len(line.Comments.Before[0].Token) == 0 { + line.Comments.Before = line.Comments.Before[:0] + } + if len(line.Token) >= 2 { // example.com v1.2.3 + line.Token[1] = v + } + } else { + if len(line.Token) >= 3 { // require example.com v1.2.3 + line.Token[2] = v + } + } + } +} + +// setIndirect sets line to have (or not have) a "// indirect" comment. +func (r *Require) setIndirect(indirect bool) { + r.Indirect = indirect + line := r.Syntax + if isIndirect(line) == indirect { + return + } + if indirect { + // Adding comment. + if len(line.Suffix) == 0 { + // New comment. + line.Suffix = []Comment{{Token: "// indirect", Suffix: true}} + return + } + + com := &line.Suffix[0] + text := strings.TrimSpace(strings.TrimPrefix(com.Token, string(slashSlash))) + if text == "" { + // Empty comment. + com.Token = "// indirect" + return + } + + // Insert at beginning of existing comment. + com.Token = "// indirect; " + text + return + } + + // Removing comment. + f := strings.TrimSpace(strings.TrimPrefix(line.Suffix[0].Token, string(slashSlash))) + if f == "indirect" { + // Remove whole comment. + line.Suffix = nil + return + } + + // Remove comment prefix. + com := &line.Suffix[0] + i := strings.Index(com.Token, "indirect;") + com.Token = "//" + com.Token[i+len("indirect;"):] +} + +// isIndirect reports whether line has a "// indirect" comment, +// meaning it is in go.mod only for its effect on indirect dependencies, +// so that it can be dropped entirely once the effective version of the +// indirect dependency reaches the given minimum version. +func isIndirect(line *Line) bool { + if len(line.Suffix) == 0 { + return false + } + f := strings.Fields(strings.TrimPrefix(line.Suffix[0].Token, string(slashSlash))) + return (len(f) == 1 && f[0] == "indirect" || len(f) > 1 && f[0] == "indirect;") +} + +func (f *File) AddModuleStmt(path string) error { + if f.Syntax == nil { + f.Syntax = new(FileSyntax) + } + if f.Module == nil { + f.Module = &Module{ + Mod: module.Version{Path: path}, + Syntax: f.Syntax.addLine(nil, "module", AutoQuote(path)), + } + } else { + f.Module.Mod.Path = path + f.Syntax.updateLine(f.Module.Syntax, "module", AutoQuote(path)) + } + return nil +} + +func (f *File) AddComment(text string) { + if f.Syntax == nil { + f.Syntax = new(FileSyntax) + } + f.Syntax.Stmt = append(f.Syntax.Stmt, &CommentBlock{ + Comments: Comments{ + Before: []Comment{ + { + Token: text, + }, + }, + }, + }) +} + +type VersionFixer func(path, version string) (string, error) + +// errDontFix is returned by a VersionFixer to indicate the version should be +// left alone, even if it's not canonical. +var dontFixRetract VersionFixer = func(_, vers string) (string, error) { + return vers, nil +} + +// Parse parses and returns a go.mod file. +// +// file is the name of the file, used in positions and errors. +// +// data is the content of the file. +// +// fix is an optional function that canonicalizes module versions. +// If fix is nil, all module versions must be canonical ([module.CanonicalVersion] +// must return the same string). +func Parse(file string, data []byte, fix VersionFixer) (*File, error) { + return parseToFile(file, data, fix, true) +} + +// ParseLax is like Parse but ignores unknown statements. +// It is used when parsing go.mod files other than the main module, +// under the theory that most statement types we add in the future will +// only apply in the main module, like exclude and replace, +// and so we get better gradual deployments if old go commands +// simply ignore those statements when found in go.mod files +// in dependencies. +func ParseLax(file string, data []byte, fix VersionFixer) (*File, error) { + return parseToFile(file, data, fix, false) +} + +func parseToFile(file string, data []byte, fix VersionFixer, strict bool) (parsed *File, err error) { + fs, err := parse(file, data) + if err != nil { + return nil, err + } + f := &File{ + Syntax: fs, + } + var errs ErrorList + + // fix versions in retract directives after the file is parsed. + // We need the module path to fix versions, and it might be at the end. + defer func() { + oldLen := len(errs) + f.fixRetract(fix, &errs) + if len(errs) > oldLen { + parsed, err = nil, errs + } + }() + + for _, x := range fs.Stmt { + switch x := x.(type) { + case *Line: + f.add(&errs, nil, x, x.Token[0], x.Token[1:], fix, strict) + + case *LineBlock: + if len(x.Token) > 1 { + if strict { + errs = append(errs, Error{ + Filename: file, + Pos: x.Start, + Err: fmt.Errorf("unknown block type: %s", strings.Join(x.Token, " ")), + }) + } + continue + } + switch x.Token[0] { + default: + if strict { + errs = append(errs, Error{ + Filename: file, + Pos: x.Start, + Err: fmt.Errorf("unknown block type: %s", strings.Join(x.Token, " ")), + }) + } + continue + case "module", "godebug", "require", "exclude", "replace", "retract", "tool": + for _, l := range x.Line { + f.add(&errs, x, l, x.Token[0], l.Token, fix, strict) + } + } + } + } + + if len(errs) > 0 { + return nil, errs + } + return f, nil +} + +var GoVersionRE = lazyregexp.New(`^([1-9][0-9]*)\.(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))?([a-z]+[0-9]+)?$`) +var laxGoVersionRE = lazyregexp.New(`^v?(([1-9][0-9]*)\.(0|[1-9][0-9]*))([^0-9].*)$`) + +// Toolchains must be named beginning with `go1`, +// like "go1.20.3" or "go1.20.3-gccgo". As a special case, "default" is also permitted. +// Note that this regexp is a much looser condition than go/version.IsValid, +// for forward compatibility. +// (This code has to be work to identify new toolchains even if we tweak the syntax in the future.) +var ToolchainRE = lazyregexp.New(`^default$|^go1($|\.)`) + +func (f *File) add(errs *ErrorList, block *LineBlock, line *Line, verb string, args []string, fix VersionFixer, strict bool) { + // If strict is false, this module is a dependency. + // We ignore all unknown directives as well as main-module-only + // directives like replace and exclude. It will work better for + // forward compatibility if we can depend on modules that have unknown + // statements (presumed relevant only when acting as the main module) + // and simply ignore those statements. + if !strict { + switch verb { + case "go", "module", "retract", "require": + // want these even for dependency go.mods + default: + return + } + } + + wrapModPathError := func(modPath string, err error) { + *errs = append(*errs, Error{ + Filename: f.Syntax.Name, + Pos: line.Start, + ModPath: modPath, + Verb: verb, + Err: err, + }) + } + wrapError := func(err error) { + *errs = append(*errs, Error{ + Filename: f.Syntax.Name, + Pos: line.Start, + Err: err, + }) + } + errorf := func(format string, args ...interface{}) { + wrapError(fmt.Errorf(format, args...)) + } + + switch verb { + default: + errorf("unknown directive: %s", verb) + + case "go": + if f.Go != nil { + errorf("repeated go statement") + return + } + if len(args) != 1 { + errorf("go directive expects exactly one argument") + return + } else if !GoVersionRE.MatchString(args[0]) { + fixed := false + if !strict { + if m := laxGoVersionRE.FindStringSubmatch(args[0]); m != nil { + args[0] = m[1] + fixed = true + } + } + if !fixed { + errorf("invalid go version '%s': must match format 1.23.0", args[0]) + return + } + } + + f.Go = &Go{Syntax: line} + f.Go.Version = args[0] + + case "toolchain": + if f.Toolchain != nil { + errorf("repeated toolchain statement") + return + } + if len(args) != 1 { + errorf("toolchain directive expects exactly one argument") + return + } else if !ToolchainRE.MatchString(args[0]) { + errorf("invalid toolchain version '%s': must match format go1.23.0 or default", args[0]) + return + } + f.Toolchain = &Toolchain{Syntax: line} + f.Toolchain.Name = args[0] + + case "module": + if f.Module != nil { + errorf("repeated module statement") + return + } + deprecated := parseDeprecation(block, line) + f.Module = &Module{ + Syntax: line, + Deprecated: deprecated, + } + if len(args) != 1 { + errorf("usage: module module/path") + return + } + s, err := parseString(&args[0]) + if err != nil { + errorf("invalid quoted string: %v", err) + return + } + f.Module.Mod = module.Version{Path: s} + + case "godebug": + if len(args) != 1 || strings.ContainsAny(args[0], "\"`',") { + errorf("usage: godebug key=value") + return + } + key, value, ok := strings.Cut(args[0], "=") + if !ok { + errorf("usage: godebug key=value") + return + } + f.Godebug = append(f.Godebug, &Godebug{ + Key: key, + Value: value, + Syntax: line, + }) + + case "require", "exclude": + if len(args) != 2 { + errorf("usage: %s module/path v1.2.3", verb) + return + } + s, err := parseString(&args[0]) + if err != nil { + errorf("invalid quoted string: %v", err) + return + } + v, err := parseVersion(verb, s, &args[1], fix) + if err != nil { + wrapError(err) + return + } + pathMajor, err := modulePathMajor(s) + if err != nil { + wrapError(err) + return + } + if err := module.CheckPathMajor(v, pathMajor); err != nil { + wrapModPathError(s, err) + return + } + if verb == "require" { + f.Require = append(f.Require, &Require{ + Mod: module.Version{Path: s, Version: v}, + Syntax: line, + Indirect: isIndirect(line), + }) + } else { + f.Exclude = append(f.Exclude, &Exclude{ + Mod: module.Version{Path: s, Version: v}, + Syntax: line, + }) + } + + case "replace": + replace, wrappederr := parseReplace(f.Syntax.Name, line, verb, args, fix) + if wrappederr != nil { + *errs = append(*errs, *wrappederr) + return + } + f.Replace = append(f.Replace, replace) + + case "retract": + rationale := parseDirectiveComment(block, line) + vi, err := parseVersionInterval(verb, "", &args, dontFixRetract) + if err != nil { + if strict { + wrapError(err) + return + } else { + // Only report errors parsing intervals in the main module. We may + // support additional syntax in the future, such as open and half-open + // intervals. Those can't be supported now, because they break the + // go.mod parser, even in lax mode. + return + } + } + if len(args) > 0 && strict { + // In the future, there may be additional information after the version. + errorf("unexpected token after version: %q", args[0]) + return + } + retract := &Retract{ + VersionInterval: vi, + Rationale: rationale, + Syntax: line, + } + f.Retract = append(f.Retract, retract) + + case "tool": + if len(args) != 1 { + errorf("tool directive expects exactly one argument") + return + } + s, err := parseString(&args[0]) + if err != nil { + errorf("invalid quoted string: %v", err) + return + } + f.Tool = append(f.Tool, &Tool{ + Path: s, + Syntax: line, + }) + } +} + +func parseReplace(filename string, line *Line, verb string, args []string, fix VersionFixer) (*Replace, *Error) { + wrapModPathError := func(modPath string, err error) *Error { + return &Error{ + Filename: filename, + Pos: line.Start, + ModPath: modPath, + Verb: verb, + Err: err, + } + } + wrapError := func(err error) *Error { + return &Error{ + Filename: filename, + Pos: line.Start, + Err: err, + } + } + errorf := func(format string, args ...interface{}) *Error { + return wrapError(fmt.Errorf(format, args...)) + } + + arrow := 2 + if len(args) >= 2 && args[1] == "=>" { + arrow = 1 + } + if len(args) < arrow+2 || len(args) > arrow+3 || args[arrow] != "=>" { + return nil, errorf("usage: %s module/path [v1.2.3] => other/module v1.4\n\t or %s module/path [v1.2.3] => ../local/directory", verb, verb) + } + s, err := parseString(&args[0]) + if err != nil { + return nil, errorf("invalid quoted string: %v", err) + } + pathMajor, err := modulePathMajor(s) + if err != nil { + return nil, wrapModPathError(s, err) + + } + var v string + if arrow == 2 { + v, err = parseVersion(verb, s, &args[1], fix) + if err != nil { + return nil, wrapError(err) + } + if err := module.CheckPathMajor(v, pathMajor); err != nil { + return nil, wrapModPathError(s, err) + } + } + ns, err := parseString(&args[arrow+1]) + if err != nil { + return nil, errorf("invalid quoted string: %v", err) + } + nv := "" + if len(args) == arrow+2 { + if !IsDirectoryPath(ns) { + if strings.Contains(ns, "@") { + return nil, errorf("replacement module must match format 'path version', not 'path@version'") + } + return nil, errorf("replacement module without version must be directory path (rooted or starting with . or ..)") + } + if filepath.Separator == '/' && strings.Contains(ns, `\`) { + return nil, errorf("replacement directory appears to be Windows path (on a non-windows system)") + } + } + if len(args) == arrow+3 { + nv, err = parseVersion(verb, ns, &args[arrow+2], fix) + if err != nil { + return nil, wrapError(err) + } + if IsDirectoryPath(ns) { + return nil, errorf("replacement module directory path %q cannot have version", ns) + } + } + return &Replace{ + Old: module.Version{Path: s, Version: v}, + New: module.Version{Path: ns, Version: nv}, + Syntax: line, + }, nil +} + +// fixRetract applies fix to each retract directive in f, appending any errors +// to errs. +// +// Most versions are fixed as we parse the file, but for retract directives, +// the relevant module path is the one specified with the module directive, +// and that might appear at the end of the file (or not at all). +func (f *File) fixRetract(fix VersionFixer, errs *ErrorList) { + if fix == nil { + return + } + path := "" + if f.Module != nil { + path = f.Module.Mod.Path + } + var r *Retract + wrapError := func(err error) { + *errs = append(*errs, Error{ + Filename: f.Syntax.Name, + Pos: r.Syntax.Start, + Err: err, + }) + } + + for _, r = range f.Retract { + if path == "" { + wrapError(errors.New("no module directive found, so retract cannot be used")) + return // only print the first one of these + } + + args := r.Syntax.Token + if args[0] == "retract" { + args = args[1:] + } + vi, err := parseVersionInterval("retract", path, &args, fix) + if err != nil { + wrapError(err) + } + r.VersionInterval = vi + } +} + +func (f *WorkFile) add(errs *ErrorList, line *Line, verb string, args []string, fix VersionFixer) { + wrapError := func(err error) { + *errs = append(*errs, Error{ + Filename: f.Syntax.Name, + Pos: line.Start, + Err: err, + }) + } + errorf := func(format string, args ...interface{}) { + wrapError(fmt.Errorf(format, args...)) + } + + switch verb { + default: + errorf("unknown directive: %s", verb) + + case "go": + if f.Go != nil { + errorf("repeated go statement") + return + } + if len(args) != 1 { + errorf("go directive expects exactly one argument") + return + } else if !GoVersionRE.MatchString(args[0]) { + errorf("invalid go version '%s': must match format 1.23.0", args[0]) + return + } + + f.Go = &Go{Syntax: line} + f.Go.Version = args[0] + + case "toolchain": + if f.Toolchain != nil { + errorf("repeated toolchain statement") + return + } + if len(args) != 1 { + errorf("toolchain directive expects exactly one argument") + return + } else if !ToolchainRE.MatchString(args[0]) { + errorf("invalid toolchain version '%s': must match format go1.23.0 or default", args[0]) + return + } + + f.Toolchain = &Toolchain{Syntax: line} + f.Toolchain.Name = args[0] + + case "godebug": + if len(args) != 1 || strings.ContainsAny(args[0], "\"`',") { + errorf("usage: godebug key=value") + return + } + key, value, ok := strings.Cut(args[0], "=") + if !ok { + errorf("usage: godebug key=value") + return + } + f.Godebug = append(f.Godebug, &Godebug{ + Key: key, + Value: value, + Syntax: line, + }) + + case "use": + if len(args) != 1 { + errorf("usage: %s local/dir", verb) + return + } + s, err := parseString(&args[0]) + if err != nil { + errorf("invalid quoted string: %v", err) + return + } + f.Use = append(f.Use, &Use{ + Path: s, + Syntax: line, + }) + + case "replace": + replace, wrappederr := parseReplace(f.Syntax.Name, line, verb, args, fix) + if wrappederr != nil { + *errs = append(*errs, *wrappederr) + return + } + f.Replace = append(f.Replace, replace) + } +} + +// IsDirectoryPath reports whether the given path should be interpreted as a directory path. +// Just like on the go command line, relative paths starting with a '.' or '..' path component +// and rooted paths are directory paths; the rest are module paths. +func IsDirectoryPath(ns string) bool { + // Because go.mod files can move from one system to another, + // we check all known path syntaxes, both Unix and Windows. + return ns == "." || strings.HasPrefix(ns, "./") || strings.HasPrefix(ns, `.\`) || + ns == ".." || strings.HasPrefix(ns, "../") || strings.HasPrefix(ns, `..\`) || + strings.HasPrefix(ns, "/") || strings.HasPrefix(ns, `\`) || + len(ns) >= 2 && ('A' <= ns[0] && ns[0] <= 'Z' || 'a' <= ns[0] && ns[0] <= 'z') && ns[1] == ':' +} + +// MustQuote reports whether s must be quoted in order to appear as +// a single token in a go.mod line. +func MustQuote(s string) bool { + for _, r := range s { + switch r { + case ' ', '"', '\'', '`': + return true + + case '(', ')', '[', ']', '{', '}', ',': + if len(s) > 1 { + return true + } + + default: + if !unicode.IsPrint(r) { + return true + } + } + } + return s == "" || strings.Contains(s, "//") || strings.Contains(s, "/*") +} + +// AutoQuote returns s or, if quoting is required for s to appear in a go.mod, +// the quotation of s. +func AutoQuote(s string) string { + if MustQuote(s) { + return strconv.Quote(s) + } + return s +} + +func parseVersionInterval(verb string, path string, args *[]string, fix VersionFixer) (VersionInterval, error) { + toks := *args + if len(toks) == 0 || toks[0] == "(" { + return VersionInterval{}, fmt.Errorf("expected '[' or version") + } + if toks[0] != "[" { + v, err := parseVersion(verb, path, &toks[0], fix) + if err != nil { + return VersionInterval{}, err + } + *args = toks[1:] + return VersionInterval{Low: v, High: v}, nil + } + toks = toks[1:] + + if len(toks) == 0 { + return VersionInterval{}, fmt.Errorf("expected version after '['") + } + low, err := parseVersion(verb, path, &toks[0], fix) + if err != nil { + return VersionInterval{}, err + } + toks = toks[1:] + + if len(toks) == 0 || toks[0] != "," { + return VersionInterval{}, fmt.Errorf("expected ',' after version") + } + toks = toks[1:] + + if len(toks) == 0 { + return VersionInterval{}, fmt.Errorf("expected version after ','") + } + high, err := parseVersion(verb, path, &toks[0], fix) + if err != nil { + return VersionInterval{}, err + } + toks = toks[1:] + + if len(toks) == 0 || toks[0] != "]" { + return VersionInterval{}, fmt.Errorf("expected ']' after version") + } + toks = toks[1:] + + *args = toks + return VersionInterval{Low: low, High: high}, nil +} + +func parseString(s *string) (string, error) { + t := *s + if strings.HasPrefix(t, `"`) { + var err error + if t, err = strconv.Unquote(t); err != nil { + return "", err + } + } else if strings.ContainsAny(t, "\"'`") { + // Other quotes are reserved both for possible future expansion + // and to avoid confusion. For example if someone types 'x' + // we want that to be a syntax error and not a literal x in literal quotation marks. + return "", fmt.Errorf("unquoted string cannot contain quote") + } + *s = AutoQuote(t) + return t, nil +} + +var deprecatedRE = lazyregexp.New(`(?s)(?:^|\n\n)Deprecated: *(.*?)(?:$|\n\n)`) + +// parseDeprecation extracts the text of comments on a "module" directive and +// extracts a deprecation message from that. +// +// A deprecation message is contained in a paragraph within a block of comments +// that starts with "Deprecated:" (case sensitive). The message runs until the +// end of the paragraph and does not include the "Deprecated:" prefix. If the +// comment block has multiple paragraphs that start with "Deprecated:", +// parseDeprecation returns the message from the first. +func parseDeprecation(block *LineBlock, line *Line) string { + text := parseDirectiveComment(block, line) + m := deprecatedRE.FindStringSubmatch(text) + if m == nil { + return "" + } + return m[1] +} + +// parseDirectiveComment extracts the text of comments on a directive. +// If the directive's line does not have comments and is part of a block that +// does have comments, the block's comments are used. +func parseDirectiveComment(block *LineBlock, line *Line) string { + comments := line.Comment() + if block != nil && len(comments.Before) == 0 && len(comments.Suffix) == 0 { + comments = block.Comment() + } + groups := [][]Comment{comments.Before, comments.Suffix} + var lines []string + for _, g := range groups { + for _, c := range g { + if !strings.HasPrefix(c.Token, "//") { + continue // blank line + } + lines = append(lines, strings.TrimSpace(strings.TrimPrefix(c.Token, "//"))) + } + } + return strings.Join(lines, "\n") +} + +type ErrorList []Error + +func (e ErrorList) Error() string { + errStrs := make([]string, len(e)) + for i, err := range e { + errStrs[i] = err.Error() + } + return strings.Join(errStrs, "\n") +} + +type Error struct { + Filename string + Pos Position + Verb string + ModPath string + Err error +} + +func (e *Error) Error() string { + var pos string + if e.Pos.LineRune > 1 { + // Don't print LineRune if it's 1 (beginning of line). + // It's always 1 except in scanner errors, which are rare. + pos = fmt.Sprintf("%s:%d:%d: ", e.Filename, e.Pos.Line, e.Pos.LineRune) + } else if e.Pos.Line > 0 { + pos = fmt.Sprintf("%s:%d: ", e.Filename, e.Pos.Line) + } else if e.Filename != "" { + pos = fmt.Sprintf("%s: ", e.Filename) + } + + var directive string + if e.ModPath != "" { + directive = fmt.Sprintf("%s %s: ", e.Verb, e.ModPath) + } else if e.Verb != "" { + directive = fmt.Sprintf("%s: ", e.Verb) + } + + return pos + directive + e.Err.Error() +} + +func (e *Error) Unwrap() error { return e.Err } + +func parseVersion(verb string, path string, s *string, fix VersionFixer) (string, error) { + t, err := parseString(s) + if err != nil { + return "", &Error{ + Verb: verb, + ModPath: path, + Err: &module.InvalidVersionError{ + Version: *s, + Err: err, + }, + } + } + if fix != nil { + fixed, err := fix(path, t) + if err != nil { + if err, ok := err.(*module.ModuleError); ok { + return "", &Error{ + Verb: verb, + ModPath: path, + Err: err.Err, + } + } + return "", err + } + t = fixed + } else { + cv := module.CanonicalVersion(t) + if cv == "" { + return "", &Error{ + Verb: verb, + ModPath: path, + Err: &module.InvalidVersionError{ + Version: t, + Err: errors.New("must be of the form v1.2.3"), + }, + } + } + t = cv + } + *s = t + return *s, nil +} + +func modulePathMajor(path string) (string, error) { + _, major, ok := module.SplitPathVersion(path) + if !ok { + return "", fmt.Errorf("invalid module path") + } + return major, nil +} + +func (f *File) Format() ([]byte, error) { + return Format(f.Syntax), nil +} + +// Cleanup cleans up the file f after any edit operations. +// To avoid quadratic behavior, modifications like [File.DropRequire] +// clear the entry but do not remove it from the slice. +// Cleanup cleans out all the cleared entries. +func (f *File) Cleanup() { + w := 0 + for _, g := range f.Godebug { + if g.Key != "" { + f.Godebug[w] = g + w++ + } + } + f.Godebug = f.Godebug[:w] + + w = 0 + for _, r := range f.Require { + if r.Mod.Path != "" { + f.Require[w] = r + w++ + } + } + f.Require = f.Require[:w] + + w = 0 + for _, x := range f.Exclude { + if x.Mod.Path != "" { + f.Exclude[w] = x + w++ + } + } + f.Exclude = f.Exclude[:w] + + w = 0 + for _, r := range f.Replace { + if r.Old.Path != "" { + f.Replace[w] = r + w++ + } + } + f.Replace = f.Replace[:w] + + w = 0 + for _, r := range f.Retract { + if r.Low != "" || r.High != "" { + f.Retract[w] = r + w++ + } + } + f.Retract = f.Retract[:w] + + f.Syntax.Cleanup() +} + +func (f *File) AddGoStmt(version string) error { + if !GoVersionRE.MatchString(version) { + return fmt.Errorf("invalid language version %q", version) + } + if f.Go == nil { + var hint Expr + if f.Module != nil && f.Module.Syntax != nil { + hint = f.Module.Syntax + } else if f.Syntax == nil { + f.Syntax = new(FileSyntax) + } + f.Go = &Go{ + Version: version, + Syntax: f.Syntax.addLine(hint, "go", version), + } + } else { + f.Go.Version = version + f.Syntax.updateLine(f.Go.Syntax, "go", version) + } + return nil +} + +// DropGoStmt deletes the go statement from the file. +func (f *File) DropGoStmt() { + if f.Go != nil { + f.Go.Syntax.markRemoved() + f.Go = nil + } +} + +// DropToolchainStmt deletes the toolchain statement from the file. +func (f *File) DropToolchainStmt() { + if f.Toolchain != nil { + f.Toolchain.Syntax.markRemoved() + f.Toolchain = nil + } +} + +func (f *File) AddToolchainStmt(name string) error { + if !ToolchainRE.MatchString(name) { + return fmt.Errorf("invalid toolchain name %q", name) + } + if f.Toolchain == nil { + var hint Expr + if f.Go != nil && f.Go.Syntax != nil { + hint = f.Go.Syntax + } else if f.Module != nil && f.Module.Syntax != nil { + hint = f.Module.Syntax + } + f.Toolchain = &Toolchain{ + Name: name, + Syntax: f.Syntax.addLine(hint, "toolchain", name), + } + } else { + f.Toolchain.Name = name + f.Syntax.updateLine(f.Toolchain.Syntax, "toolchain", name) + } + return nil +} + +// AddGodebug sets the first godebug line for key to value, +// preserving any existing comments for that line and removing all +// other godebug lines for key. +// +// If no line currently exists for key, AddGodebug adds a new line +// at the end of the last godebug block. +func (f *File) AddGodebug(key, value string) error { + need := true + for _, g := range f.Godebug { + if g.Key == key { + if need { + g.Value = value + f.Syntax.updateLine(g.Syntax, "godebug", key+"="+value) + need = false + } else { + g.Syntax.markRemoved() + *g = Godebug{} + } + } + } + + if need { + f.addNewGodebug(key, value) + } + return nil +} + +// addNewGodebug adds a new godebug key=value line at the end +// of the last godebug block, regardless of any existing godebug lines for key. +func (f *File) addNewGodebug(key, value string) { + line := f.Syntax.addLine(nil, "godebug", key+"="+value) + g := &Godebug{ + Key: key, + Value: value, + Syntax: line, + } + f.Godebug = append(f.Godebug, g) +} + +// AddRequire sets the first require line for path to version vers, +// preserving any existing comments for that line and removing all +// other lines for path. +// +// If no line currently exists for path, AddRequire adds a new line +// at the end of the last require block. +func (f *File) AddRequire(path, vers string) error { + need := true + for _, r := range f.Require { + if r.Mod.Path == path { + if need { + r.Mod.Version = vers + f.Syntax.updateLine(r.Syntax, "require", AutoQuote(path), vers) + need = false + } else { + r.Syntax.markRemoved() + *r = Require{} + } + } + } + + if need { + f.AddNewRequire(path, vers, false) + } + return nil +} + +// AddNewRequire adds a new require line for path at version vers at the end of +// the last require block, regardless of any existing require lines for path. +func (f *File) AddNewRequire(path, vers string, indirect bool) { + line := f.Syntax.addLine(nil, "require", AutoQuote(path), vers) + r := &Require{ + Mod: module.Version{Path: path, Version: vers}, + Syntax: line, + } + r.setIndirect(indirect) + f.Require = append(f.Require, r) +} + +// SetRequire updates the requirements of f to contain exactly req, preserving +// the existing block structure and line comment contents (except for 'indirect' +// markings) for the first requirement on each named module path. +// +// The Syntax field is ignored for the requirements in req. +// +// Any requirements not already present in the file are added to the block +// containing the last require line. +// +// The requirements in req must specify at most one distinct version for each +// module path. +// +// If any existing requirements may be removed, the caller should call +// [File.Cleanup] after all edits are complete. +func (f *File) SetRequire(req []*Require) { + type elem struct { + version string + indirect bool + } + need := make(map[string]elem) + for _, r := range req { + if prev, dup := need[r.Mod.Path]; dup && prev.version != r.Mod.Version { + panic(fmt.Errorf("SetRequire called with conflicting versions for path %s (%s and %s)", r.Mod.Path, prev.version, r.Mod.Version)) + } + need[r.Mod.Path] = elem{r.Mod.Version, r.Indirect} + } + + // Update or delete the existing Require entries to preserve + // only the first for each module path in req. + for _, r := range f.Require { + e, ok := need[r.Mod.Path] + if ok { + r.setVersion(e.version) + r.setIndirect(e.indirect) + } else { + r.markRemoved() + } + delete(need, r.Mod.Path) + } + + // Add new entries in the last block of the file for any paths that weren't + // already present. + // + // This step is nondeterministic, but the final result will be deterministic + // because we will sort the block. + for path, e := range need { + f.AddNewRequire(path, e.version, e.indirect) + } + + f.SortBlocks() +} + +// SetRequireSeparateIndirect updates the requirements of f to contain the given +// requirements. Comment contents (except for 'indirect' markings) are retained +// from the first existing requirement for each module path. Like SetRequire, +// SetRequireSeparateIndirect adds requirements for new paths in req, +// updates the version and "// indirect" comment on existing requirements, +// and deletes requirements on paths not in req. Existing duplicate requirements +// are deleted. +// +// As its name suggests, SetRequireSeparateIndirect puts direct and indirect +// requirements into two separate blocks, one containing only direct +// requirements, and the other containing only indirect requirements. +// SetRequireSeparateIndirect may move requirements between these two blocks +// when their indirect markings change. However, SetRequireSeparateIndirect +// won't move requirements from other blocks, especially blocks with comments. +// +// If the file initially has one uncommented block of requirements, +// SetRequireSeparateIndirect will split it into a direct-only and indirect-only +// block. This aids in the transition to separate blocks. +func (f *File) SetRequireSeparateIndirect(req []*Require) { + // hasComments returns whether a line or block has comments + // other than "indirect". + hasComments := func(c Comments) bool { + return len(c.Before) > 0 || len(c.After) > 0 || len(c.Suffix) > 1 || + (len(c.Suffix) == 1 && + strings.TrimSpace(strings.TrimPrefix(c.Suffix[0].Token, string(slashSlash))) != "indirect") + } + + // moveReq adds r to block. If r was in another block, moveReq deletes + // it from that block and transfers its comments. + moveReq := func(r *Require, block *LineBlock) { + var line *Line + if r.Syntax == nil { + line = &Line{Token: []string{AutoQuote(r.Mod.Path), r.Mod.Version}} + r.Syntax = line + if r.Indirect { + r.setIndirect(true) + } + } else { + line = new(Line) + *line = *r.Syntax + if !line.InBlock && len(line.Token) > 0 && line.Token[0] == "require" { + line.Token = line.Token[1:] + } + r.Syntax.Token = nil // Cleanup will delete the old line. + r.Syntax = line + } + line.InBlock = true + block.Line = append(block.Line, line) + } + + // Examine existing require lines and blocks. + var ( + // We may insert new requirements into the last uncommented + // direct-only and indirect-only blocks. We may also move requirements + // to the opposite block if their indirect markings change. + lastDirectIndex = -1 + lastIndirectIndex = -1 + + // If there are no direct-only or indirect-only blocks, a new block may + // be inserted after the last require line or block. + lastRequireIndex = -1 + + // If there's only one require line or block, and it's uncommented, + // we'll move its requirements to the direct-only or indirect-only blocks. + requireLineOrBlockCount = 0 + + // Track the block each requirement belongs to (if any) so we can + // move them later. + lineToBlock = make(map[*Line]*LineBlock) + ) + for i, stmt := range f.Syntax.Stmt { + switch stmt := stmt.(type) { + case *Line: + if len(stmt.Token) == 0 || stmt.Token[0] != "require" { + continue + } + lastRequireIndex = i + requireLineOrBlockCount++ + if !hasComments(stmt.Comments) { + if isIndirect(stmt) { + lastIndirectIndex = i + } else { + lastDirectIndex = i + } + } + + case *LineBlock: + if len(stmt.Token) == 0 || stmt.Token[0] != "require" { + continue + } + lastRequireIndex = i + requireLineOrBlockCount++ + allDirect := len(stmt.Line) > 0 && !hasComments(stmt.Comments) + allIndirect := len(stmt.Line) > 0 && !hasComments(stmt.Comments) + for _, line := range stmt.Line { + lineToBlock[line] = stmt + if hasComments(line.Comments) { + allDirect = false + allIndirect = false + } else if isIndirect(line) { + allDirect = false + } else { + allIndirect = false + } + } + if allDirect { + lastDirectIndex = i + } + if allIndirect { + lastIndirectIndex = i + } + } + } + + oneFlatUncommentedBlock := requireLineOrBlockCount == 1 && + !hasComments(*f.Syntax.Stmt[lastRequireIndex].Comment()) + + // Create direct and indirect blocks if needed. Convert lines into blocks + // if needed. If we end up with an empty block or a one-line block, + // Cleanup will delete it or convert it to a line later. + insertBlock := func(i int) *LineBlock { + block := &LineBlock{Token: []string{"require"}} + f.Syntax.Stmt = append(f.Syntax.Stmt, nil) + copy(f.Syntax.Stmt[i+1:], f.Syntax.Stmt[i:]) + f.Syntax.Stmt[i] = block + return block + } + + ensureBlock := func(i int) *LineBlock { + switch stmt := f.Syntax.Stmt[i].(type) { + case *LineBlock: + return stmt + case *Line: + block := &LineBlock{ + Token: []string{"require"}, + Line: []*Line{stmt}, + } + stmt.Token = stmt.Token[1:] // remove "require" + stmt.InBlock = true + f.Syntax.Stmt[i] = block + return block + default: + panic(fmt.Sprintf("unexpected statement: %v", stmt)) + } + } + + var lastDirectBlock *LineBlock + if lastDirectIndex < 0 { + if lastIndirectIndex >= 0 { + lastDirectIndex = lastIndirectIndex + lastIndirectIndex++ + } else if lastRequireIndex >= 0 { + lastDirectIndex = lastRequireIndex + 1 + } else { + lastDirectIndex = len(f.Syntax.Stmt) + } + lastDirectBlock = insertBlock(lastDirectIndex) + } else { + lastDirectBlock = ensureBlock(lastDirectIndex) + } + + var lastIndirectBlock *LineBlock + if lastIndirectIndex < 0 { + lastIndirectIndex = lastDirectIndex + 1 + lastIndirectBlock = insertBlock(lastIndirectIndex) + } else { + lastIndirectBlock = ensureBlock(lastIndirectIndex) + } + + // Delete requirements we don't want anymore. + // Update versions and indirect comments on requirements we want to keep. + // If a requirement is in last{Direct,Indirect}Block with the wrong + // indirect marking after this, or if the requirement is in an single + // uncommented mixed block (oneFlatUncommentedBlock), move it to the + // correct block. + // + // Some blocks may be empty after this. Cleanup will remove them. + need := make(map[string]*Require) + for _, r := range req { + need[r.Mod.Path] = r + } + have := make(map[string]*Require) + for _, r := range f.Require { + path := r.Mod.Path + if need[path] == nil || have[path] != nil { + // Requirement not needed, or duplicate requirement. Delete. + r.markRemoved() + continue + } + have[r.Mod.Path] = r + r.setVersion(need[path].Mod.Version) + r.setIndirect(need[path].Indirect) + if need[path].Indirect && + (oneFlatUncommentedBlock || lineToBlock[r.Syntax] == lastDirectBlock) { + moveReq(r, lastIndirectBlock) + } else if !need[path].Indirect && + (oneFlatUncommentedBlock || lineToBlock[r.Syntax] == lastIndirectBlock) { + moveReq(r, lastDirectBlock) + } + } + + // Add new requirements. + for path, r := range need { + if have[path] == nil { + if r.Indirect { + moveReq(r, lastIndirectBlock) + } else { + moveReq(r, lastDirectBlock) + } + f.Require = append(f.Require, r) + } + } + + f.SortBlocks() +} + +func (f *File) DropGodebug(key string) error { + for _, g := range f.Godebug { + if g.Key == key { + g.Syntax.markRemoved() + *g = Godebug{} + } + } + return nil +} + +func (f *File) DropRequire(path string) error { + for _, r := range f.Require { + if r.Mod.Path == path { + r.Syntax.markRemoved() + *r = Require{} + } + } + return nil +} + +// AddExclude adds a exclude statement to the mod file. Errors if the provided +// version is not a canonical version string +func (f *File) AddExclude(path, vers string) error { + if err := checkCanonicalVersion(path, vers); err != nil { + return err + } + + var hint *Line + for _, x := range f.Exclude { + if x.Mod.Path == path && x.Mod.Version == vers { + return nil + } + if x.Mod.Path == path { + hint = x.Syntax + } + } + + f.Exclude = append(f.Exclude, &Exclude{Mod: module.Version{Path: path, Version: vers}, Syntax: f.Syntax.addLine(hint, "exclude", AutoQuote(path), vers)}) + return nil +} + +func (f *File) DropExclude(path, vers string) error { + for _, x := range f.Exclude { + if x.Mod.Path == path && x.Mod.Version == vers { + x.Syntax.markRemoved() + *x = Exclude{} + } + } + return nil +} + +func (f *File) AddReplace(oldPath, oldVers, newPath, newVers string) error { + return addReplace(f.Syntax, &f.Replace, oldPath, oldVers, newPath, newVers) +} + +func addReplace(syntax *FileSyntax, replace *[]*Replace, oldPath, oldVers, newPath, newVers string) error { + need := true + old := module.Version{Path: oldPath, Version: oldVers} + new := module.Version{Path: newPath, Version: newVers} + tokens := []string{"replace", AutoQuote(oldPath)} + if oldVers != "" { + tokens = append(tokens, oldVers) + } + tokens = append(tokens, "=>", AutoQuote(newPath)) + if newVers != "" { + tokens = append(tokens, newVers) + } + + var hint *Line + for _, r := range *replace { + if r.Old.Path == oldPath && (oldVers == "" || r.Old.Version == oldVers) { + if need { + // Found replacement for old; update to use new. + r.New = new + syntax.updateLine(r.Syntax, tokens...) + need = false + continue + } + // Already added; delete other replacements for same. + r.Syntax.markRemoved() + *r = Replace{} + } + if r.Old.Path == oldPath { + hint = r.Syntax + } + } + if need { + *replace = append(*replace, &Replace{Old: old, New: new, Syntax: syntax.addLine(hint, tokens...)}) + } + return nil +} + +func (f *File) DropReplace(oldPath, oldVers string) error { + for _, r := range f.Replace { + if r.Old.Path == oldPath && r.Old.Version == oldVers { + r.Syntax.markRemoved() + *r = Replace{} + } + } + return nil +} + +// AddRetract adds a retract statement to the mod file. Errors if the provided +// version interval does not consist of canonical version strings +func (f *File) AddRetract(vi VersionInterval, rationale string) error { + var path string + if f.Module != nil { + path = f.Module.Mod.Path + } + if err := checkCanonicalVersion(path, vi.High); err != nil { + return err + } + if err := checkCanonicalVersion(path, vi.Low); err != nil { + return err + } + + r := &Retract{ + VersionInterval: vi, + } + if vi.Low == vi.High { + r.Syntax = f.Syntax.addLine(nil, "retract", AutoQuote(vi.Low)) + } else { + r.Syntax = f.Syntax.addLine(nil, "retract", "[", AutoQuote(vi.Low), ",", AutoQuote(vi.High), "]") + } + if rationale != "" { + for _, line := range strings.Split(rationale, "\n") { + com := Comment{Token: "// " + line} + r.Syntax.Comment().Before = append(r.Syntax.Comment().Before, com) + } + } + return nil +} + +func (f *File) DropRetract(vi VersionInterval) error { + for _, r := range f.Retract { + if r.VersionInterval == vi { + r.Syntax.markRemoved() + *r = Retract{} + } + } + return nil +} + +// AddTool adds a new tool directive with the given path. +// It does nothing if the tool line already exists. +func (f *File) AddTool(path string) error { + for _, t := range f.Tool { + if t.Path == path { + return nil + } + } + + f.Tool = append(f.Tool, &Tool{ + Path: path, + Syntax: f.Syntax.addLine(nil, "tool", path), + }) + + f.SortBlocks() + return nil +} + +// RemoveTool removes a tool directive with the given path. +// It does nothing if no such tool directive exists. +func (f *File) DropTool(path string) error { + for _, t := range f.Tool { + if t.Path == path { + t.Syntax.markRemoved() + *t = Tool{} + } + } + return nil +} + +func (f *File) SortBlocks() { + f.removeDups() // otherwise sorting is unsafe + + // semanticSortForExcludeVersionV is the Go version (plus leading "v") at which + // lines in exclude blocks start to use semantic sort instead of lexicographic sort. + // See go.dev/issue/60028. + const semanticSortForExcludeVersionV = "v1.21" + useSemanticSortForExclude := f.Go != nil && semver.Compare("v"+f.Go.Version, semanticSortForExcludeVersionV) >= 0 + + for _, stmt := range f.Syntax.Stmt { + block, ok := stmt.(*LineBlock) + if !ok { + continue + } + less := lineLess + if block.Token[0] == "exclude" && useSemanticSortForExclude { + less = lineExcludeLess + } else if block.Token[0] == "retract" { + less = lineRetractLess + } + sort.SliceStable(block.Line, func(i, j int) bool { + return less(block.Line[i], block.Line[j]) + }) + } +} + +// removeDups removes duplicate exclude, replace and tool directives. +// +// Earlier exclude and tool directives take priority. +// +// Later replace directives take priority. +// +// require directives are not de-duplicated. That's left up to higher-level +// logic (MVS). +// +// retract directives are not de-duplicated since comments are +// meaningful, and versions may be retracted multiple times. +func (f *File) removeDups() { + removeDups(f.Syntax, &f.Exclude, &f.Replace, &f.Tool) +} + +func removeDups(syntax *FileSyntax, exclude *[]*Exclude, replace *[]*Replace, tool *[]*Tool) { + kill := make(map[*Line]bool) + + // Remove duplicate excludes. + if exclude != nil { + haveExclude := make(map[module.Version]bool) + for _, x := range *exclude { + if haveExclude[x.Mod] { + kill[x.Syntax] = true + continue + } + haveExclude[x.Mod] = true + } + var excl []*Exclude + for _, x := range *exclude { + if !kill[x.Syntax] { + excl = append(excl, x) + } + } + *exclude = excl + } + + // Remove duplicate replacements. + // Later replacements take priority over earlier ones. + haveReplace := make(map[module.Version]bool) + for i := len(*replace) - 1; i >= 0; i-- { + x := (*replace)[i] + if haveReplace[x.Old] { + kill[x.Syntax] = true + continue + } + haveReplace[x.Old] = true + } + var repl []*Replace + for _, x := range *replace { + if !kill[x.Syntax] { + repl = append(repl, x) + } + } + *replace = repl + + if tool != nil { + haveTool := make(map[string]bool) + for _, t := range *tool { + if haveTool[t.Path] { + kill[t.Syntax] = true + continue + } + haveTool[t.Path] = true + } + var newTool []*Tool + for _, t := range *tool { + if !kill[t.Syntax] { + newTool = append(newTool, t) + } + } + *tool = newTool + } + + // Duplicate require and retract directives are not removed. + + // Drop killed statements from the syntax tree. + var stmts []Expr + for _, stmt := range syntax.Stmt { + switch stmt := stmt.(type) { + case *Line: + if kill[stmt] { + continue + } + case *LineBlock: + var lines []*Line + for _, line := range stmt.Line { + if !kill[line] { + lines = append(lines, line) + } + } + stmt.Line = lines + if len(lines) == 0 { + continue + } + } + stmts = append(stmts, stmt) + } + syntax.Stmt = stmts +} + +// lineLess returns whether li should be sorted before lj. It sorts +// lexicographically without assigning any special meaning to tokens. +func lineLess(li, lj *Line) bool { + for k := 0; k < len(li.Token) && k < len(lj.Token); k++ { + if li.Token[k] != lj.Token[k] { + return li.Token[k] < lj.Token[k] + } + } + return len(li.Token) < len(lj.Token) +} + +// lineExcludeLess reports whether li should be sorted before lj for lines in +// an "exclude" block. +func lineExcludeLess(li, lj *Line) bool { + if len(li.Token) != 2 || len(lj.Token) != 2 { + // Not a known exclude specification. + // Fall back to sorting lexicographically. + return lineLess(li, lj) + } + // An exclude specification has two tokens: ModulePath and Version. + // Compare module path by string order and version by semver rules. + if pi, pj := li.Token[0], lj.Token[0]; pi != pj { + return pi < pj + } + return semver.Compare(li.Token[1], lj.Token[1]) < 0 +} + +// lineRetractLess returns whether li should be sorted before lj for lines in +// a "retract" block. It treats each line as a version interval. Single versions +// are compared as if they were intervals with the same low and high version. +// Intervals are sorted in descending order, first by low version, then by +// high version, using semver.Compare. +func lineRetractLess(li, lj *Line) bool { + interval := func(l *Line) VersionInterval { + if len(l.Token) == 1 { + return VersionInterval{Low: l.Token[0], High: l.Token[0]} + } else if len(l.Token) == 5 && l.Token[0] == "[" && l.Token[2] == "," && l.Token[4] == "]" { + return VersionInterval{Low: l.Token[1], High: l.Token[3]} + } else { + // Line in unknown format. Treat as an invalid version. + return VersionInterval{} + } + } + vii := interval(li) + vij := interval(lj) + if cmp := semver.Compare(vii.Low, vij.Low); cmp != 0 { + return cmp > 0 + } + return semver.Compare(vii.High, vij.High) > 0 +} + +// checkCanonicalVersion returns a non-nil error if vers is not a canonical +// version string or does not match the major version of path. +// +// If path is non-empty, the error text suggests a format with a major version +// corresponding to the path. +func checkCanonicalVersion(path, vers string) error { + _, pathMajor, pathMajorOk := module.SplitPathVersion(path) + + if vers == "" || vers != module.CanonicalVersion(vers) { + if pathMajor == "" { + return &module.InvalidVersionError{ + Version: vers, + Err: fmt.Errorf("must be of the form v1.2.3"), + } + } + return &module.InvalidVersionError{ + Version: vers, + Err: fmt.Errorf("must be of the form %s.2.3", module.PathMajorPrefix(pathMajor)), + } + } + + if pathMajorOk { + if err := module.CheckPathMajor(vers, pathMajor); err != nil { + if pathMajor == "" { + // In this context, the user probably wrote "v2.3.4" when they meant + // "v2.3.4+incompatible". Suggest that instead of "v0 or v1". + return &module.InvalidVersionError{ + Version: vers, + Err: fmt.Errorf("should be %s+incompatible (or module %s/%v)", vers, path, semver.Major(vers)), + } + } + return err + } + } + + return nil +} diff --git a/vendor/golang.org/x/mod/modfile/work.go b/vendor/golang.org/x/mod/modfile/work.go new file mode 100644 index 00000000..5387d0c2 --- /dev/null +++ b/vendor/golang.org/x/mod/modfile/work.go @@ -0,0 +1,335 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modfile + +import ( + "fmt" + "sort" + "strings" +) + +// A WorkFile is the parsed, interpreted form of a go.work file. +type WorkFile struct { + Go *Go + Toolchain *Toolchain + Godebug []*Godebug + Use []*Use + Replace []*Replace + + Syntax *FileSyntax +} + +// A Use is a single directory statement. +type Use struct { + Path string // Use path of module. + ModulePath string // Module path in the comment. + Syntax *Line +} + +// ParseWork parses and returns a go.work file. +// +// file is the name of the file, used in positions and errors. +// +// data is the content of the file. +// +// fix is an optional function that canonicalizes module versions. +// If fix is nil, all module versions must be canonical ([module.CanonicalVersion] +// must return the same string). +func ParseWork(file string, data []byte, fix VersionFixer) (*WorkFile, error) { + fs, err := parse(file, data) + if err != nil { + return nil, err + } + f := &WorkFile{ + Syntax: fs, + } + var errs ErrorList + + for _, x := range fs.Stmt { + switch x := x.(type) { + case *Line: + f.add(&errs, x, x.Token[0], x.Token[1:], fix) + + case *LineBlock: + if len(x.Token) > 1 { + errs = append(errs, Error{ + Filename: file, + Pos: x.Start, + Err: fmt.Errorf("unknown block type: %s", strings.Join(x.Token, " ")), + }) + continue + } + switch x.Token[0] { + default: + errs = append(errs, Error{ + Filename: file, + Pos: x.Start, + Err: fmt.Errorf("unknown block type: %s", strings.Join(x.Token, " ")), + }) + continue + case "godebug", "use", "replace": + for _, l := range x.Line { + f.add(&errs, l, x.Token[0], l.Token, fix) + } + } + } + } + + if len(errs) > 0 { + return nil, errs + } + return f, nil +} + +// Cleanup cleans up the file f after any edit operations. +// To avoid quadratic behavior, modifications like [WorkFile.DropRequire] +// clear the entry but do not remove it from the slice. +// Cleanup cleans out all the cleared entries. +func (f *WorkFile) Cleanup() { + w := 0 + for _, r := range f.Use { + if r.Path != "" { + f.Use[w] = r + w++ + } + } + f.Use = f.Use[:w] + + w = 0 + for _, r := range f.Replace { + if r.Old.Path != "" { + f.Replace[w] = r + w++ + } + } + f.Replace = f.Replace[:w] + + f.Syntax.Cleanup() +} + +func (f *WorkFile) AddGoStmt(version string) error { + if !GoVersionRE.MatchString(version) { + return fmt.Errorf("invalid language version %q", version) + } + if f.Go == nil { + stmt := &Line{Token: []string{"go", version}} + f.Go = &Go{ + Version: version, + Syntax: stmt, + } + // Find the first non-comment-only block and add + // the go statement before it. That will keep file comments at the top. + i := 0 + for i = 0; i < len(f.Syntax.Stmt); i++ { + if _, ok := f.Syntax.Stmt[i].(*CommentBlock); !ok { + break + } + } + f.Syntax.Stmt = append(append(f.Syntax.Stmt[:i:i], stmt), f.Syntax.Stmt[i:]...) + } else { + f.Go.Version = version + f.Syntax.updateLine(f.Go.Syntax, "go", version) + } + return nil +} + +func (f *WorkFile) AddToolchainStmt(name string) error { + if !ToolchainRE.MatchString(name) { + return fmt.Errorf("invalid toolchain name %q", name) + } + if f.Toolchain == nil { + stmt := &Line{Token: []string{"toolchain", name}} + f.Toolchain = &Toolchain{ + Name: name, + Syntax: stmt, + } + // Find the go line and add the toolchain line after it. + // Or else find the first non-comment-only block and add + // the toolchain line before it. That will keep file comments at the top. + i := 0 + for i = 0; i < len(f.Syntax.Stmt); i++ { + if line, ok := f.Syntax.Stmt[i].(*Line); ok && len(line.Token) > 0 && line.Token[0] == "go" { + i++ + goto Found + } + } + for i = 0; i < len(f.Syntax.Stmt); i++ { + if _, ok := f.Syntax.Stmt[i].(*CommentBlock); !ok { + break + } + } + Found: + f.Syntax.Stmt = append(append(f.Syntax.Stmt[:i:i], stmt), f.Syntax.Stmt[i:]...) + } else { + f.Toolchain.Name = name + f.Syntax.updateLine(f.Toolchain.Syntax, "toolchain", name) + } + return nil +} + +// DropGoStmt deletes the go statement from the file. +func (f *WorkFile) DropGoStmt() { + if f.Go != nil { + f.Go.Syntax.markRemoved() + f.Go = nil + } +} + +// DropToolchainStmt deletes the toolchain statement from the file. +func (f *WorkFile) DropToolchainStmt() { + if f.Toolchain != nil { + f.Toolchain.Syntax.markRemoved() + f.Toolchain = nil + } +} + +// AddGodebug sets the first godebug line for key to value, +// preserving any existing comments for that line and removing all +// other godebug lines for key. +// +// If no line currently exists for key, AddGodebug adds a new line +// at the end of the last godebug block. +func (f *WorkFile) AddGodebug(key, value string) error { + need := true + for _, g := range f.Godebug { + if g.Key == key { + if need { + g.Value = value + f.Syntax.updateLine(g.Syntax, "godebug", key+"="+value) + need = false + } else { + g.Syntax.markRemoved() + *g = Godebug{} + } + } + } + + if need { + f.addNewGodebug(key, value) + } + return nil +} + +// addNewGodebug adds a new godebug key=value line at the end +// of the last godebug block, regardless of any existing godebug lines for key. +func (f *WorkFile) addNewGodebug(key, value string) { + line := f.Syntax.addLine(nil, "godebug", key+"="+value) + g := &Godebug{ + Key: key, + Value: value, + Syntax: line, + } + f.Godebug = append(f.Godebug, g) +} + +func (f *WorkFile) DropGodebug(key string) error { + for _, g := range f.Godebug { + if g.Key == key { + g.Syntax.markRemoved() + *g = Godebug{} + } + } + return nil +} + +func (f *WorkFile) AddUse(diskPath, modulePath string) error { + need := true + for _, d := range f.Use { + if d.Path == diskPath { + if need { + d.ModulePath = modulePath + f.Syntax.updateLine(d.Syntax, "use", AutoQuote(diskPath)) + need = false + } else { + d.Syntax.markRemoved() + *d = Use{} + } + } + } + + if need { + f.AddNewUse(diskPath, modulePath) + } + return nil +} + +func (f *WorkFile) AddNewUse(diskPath, modulePath string) { + line := f.Syntax.addLine(nil, "use", AutoQuote(diskPath)) + f.Use = append(f.Use, &Use{Path: diskPath, ModulePath: modulePath, Syntax: line}) +} + +func (f *WorkFile) SetUse(dirs []*Use) { + need := make(map[string]string) + for _, d := range dirs { + need[d.Path] = d.ModulePath + } + + for _, d := range f.Use { + if modulePath, ok := need[d.Path]; ok { + d.ModulePath = modulePath + } else { + d.Syntax.markRemoved() + *d = Use{} + } + } + + // TODO(#45713): Add module path to comment. + + for diskPath, modulePath := range need { + f.AddNewUse(diskPath, modulePath) + } + f.SortBlocks() +} + +func (f *WorkFile) DropUse(path string) error { + for _, d := range f.Use { + if d.Path == path { + d.Syntax.markRemoved() + *d = Use{} + } + } + return nil +} + +func (f *WorkFile) AddReplace(oldPath, oldVers, newPath, newVers string) error { + return addReplace(f.Syntax, &f.Replace, oldPath, oldVers, newPath, newVers) +} + +func (f *WorkFile) DropReplace(oldPath, oldVers string) error { + for _, r := range f.Replace { + if r.Old.Path == oldPath && r.Old.Version == oldVers { + r.Syntax.markRemoved() + *r = Replace{} + } + } + return nil +} + +func (f *WorkFile) SortBlocks() { + f.removeDups() // otherwise sorting is unsafe + + for _, stmt := range f.Syntax.Stmt { + block, ok := stmt.(*LineBlock) + if !ok { + continue + } + sort.SliceStable(block.Line, func(i, j int) bool { + return lineLess(block.Line[i], block.Line[j]) + }) + } +} + +// removeDups removes duplicate replace directives. +// +// Later replace directives take priority. +// +// require directives are not de-duplicated. That's left up to higher-level +// logic (MVS). +// +// retract directives are not de-duplicated since comments are +// meaningful, and versions may be retracted multiple times. +func (f *WorkFile) removeDups() { + removeDups(f.Syntax, nil, &f.Replace, nil) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 971edd46..ba6332f3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,9 +1,10 @@ # github.com/VividCortex/ewma v1.1.1 ## explicit github.com/VividCortex/ewma -# github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6 -## explicit; go 1.12 +# github.com/andybalholm/brotli v1.1.0 +## explicit; go 1.13 github.com/andybalholm/brotli +github.com/andybalholm/brotli/matchfinder # github.com/cheggaaa/pb/v3 v3.0.2 ## explicit github.com/cheggaaa/pb/v3 @@ -14,7 +15,7 @@ github.com/dave/jennifer/jen # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/dsnet/compress v0.0.1 +# github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 ## explicit; go 1.9 github.com/dsnet/compress github.com/dsnet/compress/bzip2 @@ -25,10 +26,6 @@ github.com/dsnet/compress/internal/prefix # github.com/fatih/color v1.13.0 ## explicit; go 1.13 github.com/fatih/color -# github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721 -## explicit -github.com/golang/gddo/httputil -github.com/golang/gddo/httputil/header # github.com/golang/snappy v0.0.4 ## explicit github.com/golang/snappy @@ -40,12 +37,14 @@ github.com/inconshreveable/mousetrap github.com/klauspost/compress github.com/klauspost/compress/flate github.com/klauspost/compress/fse +github.com/klauspost/compress/gzip github.com/klauspost/compress/huff0 github.com/klauspost/compress/internal/cpuinfo github.com/klauspost/compress/internal/snapref +github.com/klauspost/compress/zip github.com/klauspost/compress/zstd github.com/klauspost/compress/zstd/internal/xxhash -# github.com/klauspost/pgzip v1.2.1 +# github.com/klauspost/pgzip v1.2.5 ## explicit github.com/klauspost/pgzip # github.com/mattn/go-colorable v0.1.13 @@ -57,11 +56,8 @@ github.com/mattn/go-isatty # github.com/mattn/go-runewidth v0.0.4 ## explicit github.com/mattn/go-runewidth -# github.com/mholt/archiver v2.1.0+incompatible -## explicit -github.com/mholt/archiver -# github.com/mholt/archiver/v3 v3.3.0 -## explicit; go 1.12 +# github.com/mholt/archiver/v3 v3.5.1 +## explicit; go 1.13 github.com/mholt/archiver/v3 # github.com/nmiyake/pkg/dirs v1.1.0 ## explicit; go 1.16 @@ -69,7 +65,7 @@ github.com/nmiyake/pkg/dirs # github.com/nmiyake/pkg/errorstringer v1.1.0 ## explicit; go 1.16 github.com/nmiyake/pkg/errorstringer -# github.com/nwaples/rardecode v1.0.0 +# github.com/nwaples/rardecode v1.1.0 ## explicit github.com/nwaples/rardecode # github.com/palantir/conjure-go/v6 v6.60.0 @@ -96,8 +92,8 @@ github.com/palantir/distgo/pkg/git # github.com/palantir/go-ptimports/v2 v2.10.0 ## explicit; go 1.16 github.com/palantir/go-ptimports/v2/ptimports -# github.com/palantir/godel/v2 v2.114.0 -## explicit; go 1.22 +# github.com/palantir/godel/v2 v2.118.0 +## explicit; go 1.21.0 github.com/palantir/godel/v2/framework/artifactresolver github.com/palantir/godel/v2/framework/builtintasks github.com/palantir/godel/v2/framework/builtintasks/githooks @@ -132,8 +128,8 @@ github.com/palantir/pkg/cobracli # github.com/palantir/pkg/matcher v1.2.0 ## explicit; go 1.19 github.com/palantir/pkg/matcher -# github.com/palantir/pkg/pkgpath v1.2.0 -## explicit; go 1.19 +# github.com/palantir/pkg/pkgpath v1.3.0 +## explicit; go 1.21 github.com/palantir/pkg/pkgpath # github.com/palantir/pkg/safehttp v1.1.0 ## explicit; go 1.19 @@ -157,10 +153,13 @@ github.com/palantir/witchcraft-go-error/internal/errors # github.com/palantir/witchcraft-go-params v1.36.0 ## explicit; go 1.21 github.com/palantir/witchcraft-go-params -# github.com/pierrec/lz4 v2.0.5+incompatible -## explicit -github.com/pierrec/lz4 -github.com/pierrec/lz4/internal/xxh32 +# github.com/pierrec/lz4/v4 v4.1.2 +## explicit; go 1.14 +github.com/pierrec/lz4/v4 +github.com/pierrec/lz4/v4/internal/lz4block +github.com/pierrec/lz4/v4/internal/lz4errors +github.com/pierrec/lz4/v4/internal/lz4stream +github.com/pierrec/lz4/v4/internal/xxh32 # github.com/pkg/errors v0.9.1 ## explicit github.com/pkg/errors @@ -201,6 +200,7 @@ github.com/xi2/xz # golang.org/x/mod v0.20.0 ## explicit; go 1.18 golang.org/x/mod/internal/lazyregexp +golang.org/x/mod/modfile golang.org/x/mod/module golang.org/x/mod/semver # golang.org/x/sync v0.8.0