From 1fda34baa4c89afaa8880c51964a2357e4abb53e Mon Sep 17 00:00:00 2001 From: btoews Date: Wed, 8 Jan 2025 12:28:25 -0700 Subject: [PATCH 1/3] need to add apt source to install consul on latest ubuntu --- .github/workflows/push.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index e753481..224f3c6 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -41,7 +41,10 @@ jobs: go-version-file: 'go.mod' - name: apt install - run: sudo apt install -y fuse3 libsqlite3-dev consul + run: | + wget -O - https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo apt update && sudo apt install -y fuse3 libsqlite3-dev consul - name: check fuse version run: fusermount -V @@ -75,7 +78,10 @@ jobs: go-version-file: 'go.mod' - name: apt install - run: sudo apt install -y fuse3 libsqlite3-dev consul + run: | + wget -O - https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo apt update && sudo apt install -y fuse3 libsqlite3-dev consul - name: Start consul in dev mode run: consul agent -dev & From 561a6d660f2ad61f0db257ce67cfae4c6e8dd8d2 Mon Sep 17 00:00:00 2001 From: btoews Date: Wed, 8 Jan 2025 12:46:47 -0700 Subject: [PATCH 2/3] fix some staticchecks --- cmd/litefs/export_test.go | 3 ++- cmd/litefs/import_test.go | 3 ++- cmd/litefs/mount_linux.go | 3 ++- cmd/litefs/mount_test.go | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cmd/litefs/export_test.go b/cmd/litefs/export_test.go index 3483420..20b3fb3 100644 --- a/cmd/litefs/export_test.go +++ b/cmd/litefs/export_test.go @@ -1,4 +1,5 @@ -// go:build linux +//go:build linux + package main_test import ( diff --git a/cmd/litefs/import_test.go b/cmd/litefs/import_test.go index 1894bd7..0ad9e40 100644 --- a/cmd/litefs/import_test.go +++ b/cmd/litefs/import_test.go @@ -1,4 +1,5 @@ -// go:build linux +//go:build linux + package main_test import ( diff --git a/cmd/litefs/mount_linux.go b/cmd/litefs/mount_linux.go index d472c29..13690e6 100644 --- a/cmd/litefs/mount_linux.go +++ b/cmd/litefs/mount_linux.go @@ -1,4 +1,5 @@ -// go:build linux +//go:build linux + package main import ( diff --git a/cmd/litefs/mount_test.go b/cmd/litefs/mount_test.go index bf37208..8208961 100644 --- a/cmd/litefs/mount_test.go +++ b/cmd/litefs/mount_test.go @@ -1,4 +1,5 @@ -// go:build linux +//go:build linux + package main_test import ( From 3608110f93c928f2fa1d557d2c1e54e9e30cde72 Mon Sep 17 00:00:00 2001 From: btoews Date: Wed, 8 Jan 2025 12:19:47 -0700 Subject: [PATCH 3/3] use optimized checksumming when loading database --- db.go | 28 +++++++++++++++++----------- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/db.go b/db.go index ab3d591..94f3b2b 100644 --- a/db.go +++ b/db.go @@ -930,20 +930,26 @@ func (db *DB) initDatabaseFile() error { // Build per-page checksum map for existing pages. The database could be // short compared to the page count in the header so just checksum what we // can. The database may recover in applyLTX() so we'll do validation then. - buf := make([]byte, db.pageSize) db.chksums.pages = make([]ltx.Checksum, db.PageN()) db.chksums.blocks = make([]ltx.Checksum, pageChksumBlock(db.PageN())) - for pgno := uint32(1); pgno <= db.PageN(); pgno++ { - offset := int64(pgno-1) * int64(db.pageSize) - if _, err := internal.ReadFullAt(f, buf, offset); err == io.EOF || err == io.ErrUnexpectedEOF { - log.Printf("database checksum ending early at page %d of %d ", pgno-1, db.PageN()) - break - } else if err != nil { - return fmt.Errorf("read database page %d: %w", pgno, err) - } - chksum := ltx.ChecksumPage(pgno, buf) - db.setDatabasePageChecksum(pgno, chksum) + lastGoodPage, err := ltx.ChecksumPages(db.DatabasePath(), db.pageSize, db.PageN(), 0, db.chksums.pages) + + // lastGoodPage tells us how far we got before the first error. Most likely + // is that we got an EOF because the db was short, in which case no + // subsequent checksums would have been written. To be cautious though, + // zero out any checksums after the last good page. + clear(db.chksums.pages[lastGoodPage:]) + + // Always overwrite the lock page as a zero checksum. + if lockPage := ltx.LockPgno(db.pageSize); len(db.chksums.pages) >= int(lockPage) { + db.chksums.pages[lockPage-1] = 0 + } + + if err == io.EOF || err == io.ErrUnexpectedEOF { + log.Printf("database checksum ending early at page %d of %d ", lastGoodPage, db.PageN()) + } else if err != nil { + return fmt.Errorf("read database page %d: %w", lastGoodPage+1, err) } return nil diff --git a/go.mod b/go.mod index a4b57c9..1f57b2f 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.16-0.20220918133448-90900be5db1a github.com/prometheus/client_golang v1.13.0 github.com/superfly/litefs-go v0.0.0-20230227231337-34ea5dcf1e0b - github.com/superfly/ltx v0.3.13 + github.com/superfly/ltx v0.3.14 golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc golang.org/x/net v0.17.0 golang.org/x/sync v0.4.0 diff --git a/go.sum b/go.sum index fc4aa9b..36e1dc7 100644 --- a/go.sum +++ b/go.sum @@ -308,8 +308,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/superfly/litefs-go v0.0.0-20230227231337-34ea5dcf1e0b h1:+WuhtZFB8fNdPeaMUtuB/U8aknXBXdDW/mBm/HTYJNg= github.com/superfly/litefs-go v0.0.0-20230227231337-34ea5dcf1e0b/go.mod h1:h+GUx1V2s0C5nY73ZN82760eWEJrpMaiDweF31VmJKk= -github.com/superfly/ltx v0.3.13 h1:IbuocKJ6sY2jYvZbpUGMYmTkvaLSGUderEZwmaIUmJ0= -github.com/superfly/ltx v0.3.13/go.mod h1:ly+Dq7UVacQVEI5/b0r6j+PSNy9ibwx1yikcWAaSkhE= +github.com/superfly/ltx v0.3.14 h1:u/w2NJsyaAyh04+Oy2x5tQpWJzpzf9iQbY1wgfMZnW4= +github.com/superfly/ltx v0.3.14/go.mod h1:ly+Dq7UVacQVEI5/b0r6j+PSNy9ibwx1yikcWAaSkhE= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=