Merge pull request #605 from alfrunes/no-cgo

Allow using mender-artifact without `cgo`

LIC_FILES_CHKSUM.sha256

@@ -26,7 +26,6 @@ c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4  vendor/github.
 # BSD 3 Clause licenses.
 2eb550be6801c1ea434feba53bf6d12e7c71c90253e0a9de4a4f46cf88b56477  vendor/github.com/pmezard/go-difflib/LICENSE
 2d36597f7117c38b006835ae7f537487207d8ec407aa9d9980794b2030cbc067  vendor/golang.org/x/sys/LICENSE
-87640bc4df2ceb1559f268a3db1ba859ab780f7ba5b1b4545173d4680a3d918b  vendor/github.com/remyoudompheng/go-liblzma/LICENSE
 0d9e582ee4bff57bf1189c9e514e6da7ce277f9cd3bc2d488b22fbb39a6d87cf  vendor/github.com/klauspost/compress/LICENSE
 75e1ca97a84a9da6051dee0114333388216f2c4a5a028296b882ff3d57274735  vendor/github.com/russross/blackfriday/v2/LICENSE.txt
 17b5d209ba8f9684257ecfcff87df6ceda6194143a8fbd074f29727cff6f0c40  vendor/github.com/google/go-cmp/LICENSE
@@ -43,6 +42,7 @@ b95218cd9607855a6536384c0262922b30a0c2bf56e4ced790240f3a3bac4722  vendor/github.
 2d36597f7117c38b006835ae7f537487207d8ec407aa9d9980794b2030cbc067  vendor/golang.org/x/crypto/LICENSE
 2d36597f7117c38b006835ae7f537487207d8ec407aa9d9980794b2030cbc067  vendor/golang.org/x/time/LICENSE
 f69f157b0be75da373605dbc8bbf142e8924ee82d8f44f11bcaf351335bf98cf  vendor/github.com/klauspost/compress/internal/snapref/LICENSE
+701cbcc144a7c865b072f27547e66ddbf17ac44a3bd21f71149ace02518f107c  vendor/github.com/ulikunitz/xz/LICENSE
 #
 # ISC licenses.
 1b93a317849ee09d3d7e4f1d20c2b78ddb230b4becb12d7c224c927b9d470251  vendor/github.com/davecgh/go-spew/LICENSE

Makefile

@@ -2,14 +2,16 @@ GO ?= go
 GOFMT ?= gofmt
 V ?=
 PREFIX ?= /usr/local
-PKGS = $(shell go list ./... | grep -v vendor)
-SUBPKGS = $(shell go list ./... | sed '1d' | tr '\n' ',' | sed 's/,$$//1')
+PKGS = $(shell go list ./... | tr '\n' ',' | head -c -1)
 PKGNAME = mender-artifact
 PKGFILES = $(shell find . \( -path ./vendor -o -path ./Godeps \) -prune \
 		-o -type f -name '*.go' -print)
 PKGFILES_notest = $(shell echo $(PKGFILES) | tr ' ' '\n' | grep -v _test.go)
 GOCYCLO ?= 20
 
+GOARCH ?= $(shell go env GOARCH)
+GOOS ?= $(shell go env GOOS)
+
 CGO_ENABLED=1
 export CGO_ENABLED
 
@@ -28,39 +30,38 @@ ifeq ($(V),1)
 BUILDV = -v
 endif
 
-TAGS =
-ifeq ($(LOCAL),1)
-TAGS += local
-endif
-
-ifneq ($(TAGS),)
-BUILDTAGS = -tags '$(TAGS)'
+TAGS ?=
+ifneq ($(GOOS),linux)
+	TAGS += nopkcs11
 endif
 
 build:
-	$(GO) build $(GO_LDFLAGS) $(BUILDV) $(BUILDTAGS)
+	$(GO) build $(GO_LDFLAGS) $(BUILDV) -tags '$(TAGS)'
 
 PLATFORMS := darwin linux windows
 
-GO_LDFLAGS_WIN = -ldflags "-X github.com/mendersoftware/mender-artifact/cli.Version=$(VERSION) -linkmode=internal -s -w -extldflags '-static' -extld=x86_64-w64-mingw32-gcc"
+$(PKGNAME)-%:
+	env CGO_ENABLED=$(CGO_ENABLED) \
+		GOARCH=$(GOARCH) \
+		GOOS=$(GOOS) \
+		go build \
+		    -a $(GO_LDFLAGS) $(BUILDV) -tags '$(TAGS)' \
+		    -o $@
 
-build-native-linux:
-	 @arch="amd64";
-	 @echo "building linux";
-	 @env GOOS=linux GOARCH=$$arch \
-        $(GO) build -a $(GO_LDFLAGS) $(BUILDV) $(BUILDTAGS) -o $(PKGNAME)-linux ;
+.nopkcs11:
+	$(warning "WARNING: Building without pkcs11 support")
 
-build-native-mac:
-	@arch="amd64";
-	@echo "building mac";
-	@env GOOS=darwin GOARCH=$$arch CGO_ENABLED=0 \
-        $(GO) build -a $(GO_LDFLAGS) $(BUILDV) $(BUILDTAGS) -o $(PKGNAME)-darwin ;
+build-native-linux: $(PKGNAME)-linux
 
-build-native-windows:
-	@arch="amd64";
-	@echo "building windows";
-	@env GOOS=windows GOARCH=$$arch CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ \
-        $(GO) build $(GO_LDFLAGS_WIN) $(BUILDV) -tags $(TAGS) nolzma -o $(PKGNAME)-windows.exe ;
+build-native-mac: GOOS = darwin
+build-native-mac: TAGS = nopkcs11
+build-native-mac: CGO_ENABLED = 0
+build-native-mac: .nopkcs11 $(PKGNAME)-darwin
+
+build-native-windows: GOOS = windows
+build-native-windows: TAGS = nopkcs11
+build-native-windows: GO_LDFLAGS = -ldflags "-X github.com/mendersoftware/mender-artifact/cli.Version=44d6905 -linkmode=internal -s -w -extldflags '-static' -extld=x86_64-w64-mingw32-gcc"
+build-native-windows: .nopkcs11 $(PKGNAME)-windows
 
 build-natives: build-native-linux build-native-mac build-native-windows
 
@@ -144,15 +145,7 @@ instrument-binary:
 
 coverage:
 	rm -f coverage.txt
-	echo 'mode: count' > coverage.txt
-	set -e ; for p in $(PKGS); do \
-		rm -f coverage-tmp.txt;  \
-		$(GO) test -covermode=count -coverprofile=coverage-tmp.txt -coverpkg=$(SUBPKGS) $$p ; \
-		if [ -f coverage-tmp.txt ]; then \
-			cat coverage-tmp.txt |grep -v 'mode:' | cat >> coverage.txt; \
-		fi; \
-	done
-	rm -f coverage-tmp.txt
+	go test -tags '$(TAGS)' -covermode=atomic -coverpkg=$(PKGS) -coverprofile=coverage.txt ./...
 
 .PHONY: build clean get-tools test check \
 	cover htmlcover coverage tooldep install-autocomplete-scripts \

artifact/compressor_lzma.go

@@ -11,15 +11,16 @@
 //    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 //    See the License for the specific language governing permissions and
 //    limitations under the License.
-//go:build !nolzma && cgo
-// +build !nolzma,cgo
+//go:build !nolzma
+// +build !nolzma
 
 package artifact
 
 import (
 	"io"
 
-	xz "github.com/remyoudompheng/go-liblzma"
+	"github.com/ulikunitz/xz"
+	"github.com/ulikunitz/xz/lzma"
 )
 
 type CompressorLzma struct {
@@ -34,11 +35,34 @@ func (c *CompressorLzma) GetFileExtension() string {
 }
 
 func (c *CompressorLzma) NewReader(r io.Reader) (io.ReadCloser, error) {
-	return xz.NewReader(r)
+	r, err := rc.NewReader(r)
+	if err != nil {
+		return nil, err
+	}
+	return io.NopCloser(r), err
 }
 
+const xzDictSize = 64 * (1 << 20) // 64 MiB - dict size for preset -9
+var (
+	// Set write config as xz(1) -9
+	wc = xz.WriterConfig{
+		DictCap:  xzDictSize,
+		CheckSum: xz.CRC64,
+		// xz(1): --lzma2 mf:
+		// The default depends on the preset: 0 uses hc3, 1–3 use hc4,
+		// and the rest use bt4.
+		// bt4 = BinaryTree with 4-byte hashing
+		Matcher: lzma.HashTable4,
+		// xz(1): --block-size [...]
+		// In multi-threaded mode [...] The default size is three times
+		// the LZMA2 dictionary size or 1 MiB, whichever is more.
+		BlockSize: 3 * xzDictSize,
+	}
+	rc = xz.ReaderConfig{DictCap: xzDictSize}
+)
+
 func (c *CompressorLzma) NewWriter(w io.Writer) (io.WriteCloser, error) {
-	return xz.NewWriter(w, xz.Level9)
+	return wc.NewWriter(w)
 }
 
 func init() {

artifact/signer_unsupported.go → artifact/signer_nopkcs11.go

@@ -12,8 +12,8 @@
 //    See the License for the specific language governing permissions and
 //    limitations under the License.
 
-//go:build !linux
-// +build !linux
+//go:build nopkcs11
+// +build nopkcs11
 
 package artifact
 

artifact/signer_linux.go → artifact/signer_pkcs11.go

@@ -12,8 +12,8 @@
 //    See the License for the specific language governing permissions and
 //    limitations under the License.
 
-//go:build linux
-// +build linux
+//go:build !nopkcs11
+// +build !nopkcs11
 
 package artifact
 

go.mod

@@ -13,9 +13,9 @@ require (
 	github.com/mendersoftware/progressbar v0.0.3
 	github.com/minio/sha256-simd v1.0.1
 	github.com/pkg/errors v0.9.1
-	github.com/remyoudompheng/go-liblzma v0.0.0-20190506200333-81bf2d431b96
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.9.0
+	github.com/ulikunitz/xz v0.5.12
 	github.com/urfave/cli v1.22.15
 	golang.org/x/sys v0.19.0
 	google.golang.org/protobuf v1.33.0

go.sum

@@ -908,8 +908,6 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
-github.com/remyoudompheng/go-liblzma v0.0.0-20190506200333-81bf2d431b96 h1:J8J/cgLDRuqXJnwIrRDBvtl+LLsdg7De74znW/BRRq4=
-github.com/remyoudompheng/go-liblzma v0.0.0-20190506200333-81bf2d431b96/go.mod h1:90HvCY7+oHHUKkbeMCiHt1WuFR2/hPJ9QrljDG+v6ls=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
@@ -944,6 +942,8 @@ github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
+github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/urfave/cli v1.22.15 h1:nuqt+pdC/KqswQKhETJjo7pvn/k4xMUxgW6liI7XpnM=
 github.com/urfave/cli v1.22.15/go.mod h1:wSan1hmo5zeyLGBjRJbzRTNk8gwoYa2B9n4q9dmRIc0=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=