Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Get the page size from the second meta page if the first one is invalid #294

Merged
merged 2 commits into from
Dec 28, 2022
Merged

Get the page size from the second meta page if the first one is invalid #294

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8e6a116
get the page size from the second meta page if the first one is invalid
ahrtr Oct 18, 2021
2385046
replace all ioutil with os
ahrtr Dec 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 92 additions & 15 deletions db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,21 +252,9 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
return nil, err
}
} else {
// Read the first meta page to determine the page size.
var buf [0x1000]byte
// If we can't read the page size, but can read a page, assume
// it's the same as the OS or one given -- since that's how the
// page size was chosen in the first place.
//
// If the first page is invalid and this OS uses a different
// page size than what the database was created with then we
// are out of luck and cannot access the database.
//
// TODO: scan for next page
if bw, err := db.file.ReadAt(buf[:], 0); err == nil && bw == len(buf) {
if m := db.pageInBuffer(buf[:], 0).meta(); m.validate() == nil {
db.pageSize = int(m.pageSize)
}
// try to get the page size from the metadata pages
if pgSize, err := db.getPageSize(); err == nil {
db.pageSize = pgSize
} else {
_ = db.close()
return nil, ErrInvalid
Expand Down Expand Up @@ -309,6 +297,95 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
return db, nil
}

// getPageSize reads the pageSize from the meta pages. It tries
// to read the first meta page firstly. If the first page is invalid,
// then it tries to read the second page using the default page size.
func (db *DB) getPageSize() (int, error) {
var (
meta0CanRead, meta1CanRead bool
)

// Read the first meta page to determine the page size.
if pgSize, canRead, err := db.getPageSizeFromFirstMeta(); err != nil {
// We cannot read the page size from page 0, but can read page 0.
meta0CanRead = canRead
} else {
return pgSize, nil
}

// Read the second meta page to determine the page size.
if pgSize, canRead, err := db.getPageSizeFromSecondMeta(); err != nil {
// We cannot read the page size from page 1, but can read page 1.
meta1CanRead = canRead
} else {
return pgSize, nil
}

// If we can't read the page size from both pages, but can read
// either page, then we assume it's the same as the OS or the one
// given, since that's how the page size was chosen in the first place.
//
// If both pages are invalid, and (this OS uses a different page size
// from what the database was created with or the given page size is
// different from what the database was created with), then we are out
// of luck and cannot access the database.
if meta0CanRead || meta1CanRead {
return db.pageSize, nil
}

return 0, ErrInvalid
}

// getPageSizeFromFirstMeta reads the pageSize from the first meta page
func (db *DB) getPageSizeFromFirstMeta() (int, bool, error) {
var buf [0x1000]byte
var metaCanRead bool
if bw, err := db.file.ReadAt(buf[:], 0); err == nil && bw == len(buf) {
metaCanRead = true
if m := db.pageInBuffer(buf[:], 0).meta(); m.validate() == nil {
return int(m.pageSize), metaCanRead, nil
}
}
return 0, metaCanRead, ErrInvalid
}

// getPageSizeFromSecondMeta reads the pageSize from the second meta page
func (db *DB) getPageSizeFromSecondMeta() (int, bool, error) {
var (
fileSize int64
metaCanRead bool
)

// get the db file size
if info, err := db.file.Stat(); err != nil {
return 0, metaCanRead, err
} else {
fileSize = info.Size()
}

// We need to read the second meta page, so we should skip the first page;
// but we don't know the exact page size yet, it's chicken & egg problem.
// The solution is to try all the possible page sizes, which starts from 1KB
// and until 16MB (1024<<14) or the end of the db file
//
// TODO: should we support larger page size?
for i := 0; i <= 14; i++ {
var buf [0x1000]byte
var pos int64 = 1024 << uint(i)
if pos >= fileSize-1024 {
break
}
if bw, err := db.file.ReadAt(buf[:], pos); err == nil && bw == len(buf) {
metaCanRead = true
if m := db.pageInBuffer(buf[:], 0).meta(); m.validate() == nil {
return int(m.pageSize), metaCanRead, nil
}
}
}

return 0, metaCanRead, ErrInvalid
}

// loadFreelist reads the freelist if it is synced, or reconstructs it
// by scanning the DB if it is not synced. It assumes there are no
// concurrent accesses being made to the freelist.
Expand Down
82 changes: 82 additions & 0 deletions db_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,88 @@ func TestOpen_ErrChecksum(t *testing.T) {
}
}

// Ensure that it can read the page size from the second meta page if the first one is invalid.
// The page size is expected to be the OS's page size in this case.
func TestOpen_ReadPageSize_FromMeta1_OS(t *testing.T) {
// Create empty database.
db := MustOpenDB()
path := db.Path()
defer db.MustClose()

// Close database.
if err := db.DB.Close(); err != nil {
t.Fatal(err)
}

// Read data file.
buf, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}

// Rewrite first meta page.
meta0 := (*meta)(unsafe.Pointer(&buf[pageHeaderSize]))
meta0.pgid++
if err := os.WriteFile(path, buf, 0666); err != nil {
t.Fatal(err)
}

// Reopen data file.
if db, err := bolt.Open(path, 0666, nil); err != nil {
t.Fatalf("unexpected error: %s", err)
} else {
if db.Info().PageSize != os.Getpagesize() {
ahrtr marked this conversation as resolved.
Show resolved Hide resolved
t.Fatalf("The page size is expected to be %d, but actually is %d", os.Getpagesize(), db.Info().PageSize)
}
if err := db.Close(); err != nil {
panic(err)
}
}
}

// Ensure that it can read the page size from the second meta page if the first one is invalid.
// The page size is expected to be the given page size in this case.
func TestOpen_ReadPageSize_FromMeta1_Given(t *testing.T) {
// test page size from 1KB (1024<<0) to 16MB(1024<<14)
for i := 0; i <= 14; i++ {
givenPageSize := 1024 << uint(i)
// Create empty database.
db := MustOpenWithOption(&bolt.Options{PageSize: givenPageSize})
path := db.Path()
defer db.MustClose()

// Close database.
if err := db.DB.Close(); err != nil {
t.Fatal(err)
}

// Read data file.
buf, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}

// Rewrite meta pages.
meta0 := (*meta)(unsafe.Pointer(&buf[pageHeaderSize]))
meta0.pgid++
if err := os.WriteFile(path, buf, 0666); err != nil {
t.Fatal(err)
}

// Reopen data file.
if db, err := bolt.Open(path, 0666, &bolt.Options{PageSize: givenPageSize}); err != nil {
t.Fatalf("unexpected error: %s", err)
} else {
if db.Info().PageSize != givenPageSize {
t.Fatalf("The page size is expected to be %d, but actually is %d", givenPageSize, db.Info().PageSize)
}
if err := db.Close(); err != nil {
panic(err)
}
}
}
}

// Ensure that opening a database does not increase its size.
// https://github.com/boltdb/bolt/issues/291
func TestOpen_Size(t *testing.T) {
Expand Down