ext4

Signed-off-by: Avi Deitcher <avi@deitcher.net>

disk/disk.go

@@ -13,6 +13,7 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	"github.com/diskfs/go-diskfs/filesystem"
+	"github.com/diskfs/go-diskfs/filesystem/ext4"
 	"github.com/diskfs/go-diskfs/filesystem/fat32"
 	"github.com/diskfs/go-diskfs/filesystem/iso9660"
 	"github.com/diskfs/go-diskfs/filesystem/squashfs"
@@ -185,6 +186,8 @@ func (d *Disk) CreateFilesystem(spec FilesystemSpec) (filesystem.FileSystem, err
 		return fat32.Create(d.File, size, start, d.LogicalBlocksize, spec.VolumeLabel)
 	case filesystem.TypeISO9660:
 		return iso9660.Create(d.File, size, start, d.LogicalBlocksize, spec.WorkDir)
+	case filesystem.TypeExt4:
+		return ext4.Create(d.File, size, start, d.LogicalBlocksize, nil)
 	case filesystem.TypeSquashfs:
 		return nil, errors.New("squashfs is a read-only filesystem")
 	default:

filesystem/ext4/bitmaps.go

@@ -0,0 +1,104 @@
+package ext4
+
+import "fmt"
+
+// bitmap is a structure holding a bitmap
+type bitmap struct {
+	bits []byte
+}
+
+// bitmapFromBytes create a bitmap struct from bytes
+func bitmapFromBytes(b []byte) *bitmap {
+	// just copy them over
+	bits := make([]byte, len(b))
+	copy(bits, b)
+	bm := bitmap{
+		bits: bits,
+	}
+
+	return &bm
+}
+
+// toBytes returns raw bytes ready to be written to disk
+func (bm *bitmap) toBytes() []byte {
+	b := make([]byte, len(bm.bits))
+	copy(b, bm.bits)
+
+	return b
+}
+
+func (bm *bitmap) checkFree(location int) (bool, error) {
+	byteNumber, bitNumber := findBitForIndex(location)
+	if byteNumber > len(bm.bits) {
+		return false, fmt.Errorf("location %d is not in %d size bitmap", location, len(bm.bits)*8)
+	}
+	mask := byte(0x1) << bitNumber
+	return bm.bits[byteNumber]&mask == mask, nil
+}
+
+func (bm *bitmap) free(location int) error {
+	byteNumber, bitNumber := findBitForIndex(location)
+	if byteNumber > len(bm.bits) {
+		return fmt.Errorf("location %d is not in %d size bitmap", location, len(bm.bits)*8)
+	}
+	mask := byte(0x1) << bitNumber
+	mask = ^mask
+	bm.bits[byteNumber] &= mask
+	return nil
+}
+
+func (bm *bitmap) use(location int) error {
+	byteNumber, bitNumber := findBitForIndex(location)
+	if byteNumber > len(bm.bits) {
+		return fmt.Errorf("location %d is not in %d size bitmap", location, len(bm.bits)*8)
+	}
+	mask := byte(0x1) << bitNumber
+	bm.bits[byteNumber] |= mask
+	return nil
+}
+
+func (bm *bitmap) findFirstFree() int {
+	var location = -1
+	for i, b := range bm.bits {
+		// if all used, continue to next
+		if b&0xff == 0xff {
+			continue
+		}
+		// not all used, so find first bit set to 0
+		for j := uint8(0); j < 8; j++ {
+			mask := byte(0x1) << j
+			if b&mask != mask {
+				location = 8*i + (8 - int(j))
+				break
+			}
+		}
+		break
+	}
+	return location
+}
+
+//nolint:revive // params are unused as of yet, but will be used in the future
+func (bm *bitmap) findFirstUsed() int {
+	var location int = -1
+	for i, b := range bm.bits {
+		// if all free, continue to next
+		if b == 0x00 {
+			continue
+		}
+		// not all free, so find first bit set to 1
+		for j := uint8(0); j < 8; j++ {
+			mask := byte(0x1) << j
+			mask = ^mask
+			if b|mask != mask {
+				location = 8*i + (8 - int(j))
+				break
+			}
+		}
+		break
+	}
+	return location
+}
+
+func findBitForIndex(index int) (byteNumber int, bitNumber uint8) {
+	return index / 8, uint8(index % 8)
+}

filesystem/ext4/blockgroup.go

@@ -0,0 +1,53 @@
+package ext4
+
+import (
+	"fmt"
+)
+
+// blockGroup is a structure holding the data about a single block group
+//
+//nolint:unused // will be used in the future, not yet
+type blockGroup struct {
+	inodeBitmap    *bitmap
+	blockBitmap    *bitmap
+	blockSize      int
+	number         int
+	inodeTableSize int
+	firstDataBlock int
+}
+
+// blockGroupFromBytes create a blockGroup struct from bytes
+// it does not load the inode table or data blocks into memory, rather holding pointers to where they are
+//
+//nolint:unused // will be used in the future, not yet
+func blockGroupFromBytes(b []byte, blockSize, groupNumber int) (*blockGroup, error) {
+	expectedSize := 2 * blockSize
+	actualSize := len(b)
+	if actualSize != expectedSize {
+		return nil, fmt.Errorf("expected to be passed %d bytes for 2 blocks of size %d, instead received %d", expectedSize, blockSize, actualSize)
+	}
+	inodeBitmap := bitmapFromBytes(b[0:blockSize])
+	blockBitmap := bitmapFromBytes(b[blockSize : 2*blockSize])
+
+	bg := blockGroup{
+		inodeBitmap: inodeBitmap,
+		blockBitmap: blockBitmap,
+		number:      groupNumber,
+		blockSize:   blockSize,
+	}
+	return &bg, nil
+}
+
+// toBytes returns bitmaps ready to be written to disk
+//
+//nolint:unused // will be used in the future, not yet
+func (bg *blockGroup) toBytes() ([]byte, error) {
+	b := make([]byte, 2*bg.blockSize)
+	inodeBitmapBytes := bg.inodeBitmap.toBytes()
+	blockBitmapBytes := bg.blockBitmap.toBytes()
+
+	b = append(b, inodeBitmapBytes...)
+	b = append(b, blockBitmapBytes...)
+
+	return b, nil
+}

filesystem/ext4/checksum.go

@@ -0,0 +1,48 @@
+package ext4
+
+import (
+	"encoding/binary"
+
+	"github.com/diskfs/go-diskfs/filesystem/ext4/crc"
+)
+
+// checksumAppender is a function that takes a byte slice and returns a byte slice with a checksum appended
+type checksumAppender func([]byte) []byte
+type checksummer func([]byte) uint32
+
+// directoryChecksummer returns a function that implements checksumAppender for a directory entries block
+// original calculations can be seen for e2fsprogs https://git.kernel.org/pub/scm/fs/ext2/e2fsprogs.git/tree/lib/ext2fs/csum.c#n301
+// and in the linux tree https://github.com/torvalds/linux/blob/master/fs/ext4/namei.c#L376-L384
+func directoryChecksummer(seed, inodeNumber, inodeGeneration uint32) checksummer {
+	numBytes := make([]byte, 4)
+	binary.LittleEndian.PutUint32(numBytes, inodeNumber)
+	crcResult := crc.CRC32c(seed, numBytes)
+	genBytes := make([]byte, 4)
+	binary.LittleEndian.PutUint32(genBytes, inodeGeneration)
+	crcResult = crc.CRC32c(crcResult, genBytes)
+	return func(b []byte) uint32 {
+		checksum := crc.CRC32c(crcResult, b)
+		return checksum
+	}
+}
+
+// directoryChecksumAppender returns a function that implements checksumAppender for a directory entries block
+// original calculations can be seen for e2fsprogs https://git.kernel.org/pub/scm/fs/ext2/e2fsprogs.git/tree/lib/ext2fs/csum.c#n301
+// and in the linux tree https://github.com/torvalds/linux/blob/master/fs/ext4/namei.c#L376-L384
+func directoryChecksumAppender(seed, inodeNumber, inodeGeneration uint32) checksumAppender {
+	fn := directoryChecksummer(seed, inodeNumber, inodeGeneration)
+	return func(b []byte) []byte {
+		checksum := fn(b)
+		checksumBytes := make([]byte, 12)
+		checksumBytes[4] = 12
+		checksumBytes[7] = 0xde
+		binary.LittleEndian.PutUint32(checksumBytes[8:12], checksum)
+		b = append(b, checksumBytes...)
+		return b
+	}
+}
+
+// nullDirectoryChecksummer does not change anything
+func nullDirectoryChecksummer(b []byte) []byte {
+	return b
+}

filesystem/ext4/crc/crc16.go

@@ -0,0 +1,44 @@
+package crc
+
+var crc16tab = [256]uint16{
+	0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
+	0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
+	0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
+	0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
+	0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
+	0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
+	0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
+	0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
+	0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
+	0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
+	0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
+	0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
+	0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41,
+	0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
+	0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70,
+	0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
+	0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
+	0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
+	0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
+	0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
+	0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
+	0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
+	0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
+	0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
+	0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
+	0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3,
+	0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
+	0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
+	0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
+	0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
+	0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
+	0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0}
+
+func CRC16(crc uint16, bs []byte) uint16 {
+	l := len(bs)
+	for i := 0; i < l; i++ {
+		crc = ((crc << 8) & 0xff00) ^ crc16tab[((crc>>8)&0xff)^uint16(bs[i])]
+	}
+
+	return crc
+}

filesystem/ext4/crc/crc32.go

@@ -0,0 +1,74 @@
+package crc
+
+import (
+	"encoding/binary"
+	"hash/crc32"
+)
+
+// Define the CRC32C table using the Castagnoli polynomial
+var (
+	crc32cTable  = crc32.MakeTable(crc32.Castagnoli)
+	crc32cTables = generateTables(crc32cTable)
+)
+
+func generateTables(poly *crc32.Table) [8][256]uint32 {
+	var tab [8][256]uint32
+	tab[0] = *poly
+
+	for i := 0; i < 256; i++ {
+		crc := tab[0][i]
+		for j := 1; j < 8; j++ {
+			crc = (crc >> 8) ^ tab[0][crc&0xff]
+			tab[j][i] = crc
+		}
+	}
+
+	return tab
+}
+
+func CRC32c(base uint32, b []byte) uint32 {
+	// Compute the CRC32C checksum
+	// for reasons unknown, the checksum from go package hash/crc32, using crc32.Update(), is different from the one calculated by the kernel
+	// so we use this
+	return crc32Body(base, b, &crc32cTables)
+}
+
+// doCRC processes a single byte
+func doCRC(crc uint32, x byte, tab *[256]uint32) uint32 {
+	return tab[(crc^uint32(x))&0xff] ^ (crc >> 8)
+}
+
+// doCRC4 processes 4 bytes
+func doCRC4(q uint32, tab *[8][256]uint32) uint32 {
+	return tab[3][q&0xff] ^ tab[2][(q>>8)&0xff] ^ tab[1][(q>>16)&0xff] ^ tab[0][(q>>24)&0xff]
+}
+
+// doCRC8 processes 8 bytes
+func doCRC8(q uint32, tab *[8][256]uint32) uint32 {
+	return tab[7][q&0xff] ^ tab[6][(q>>8)&0xff] ^ tab[5][(q>>16)&0xff] ^ tab[4][(q>>24)&0xff]
+}
+
+func crc32Body(crc uint32, buf []byte, tab *[8][256]uint32) uint32 {
+	// Align it
+	for len(buf) > 0 && (uintptr(len(buf))&3) != 0 {
+		crc = doCRC(crc, buf[0], &tab[0])
+		buf = buf[1:]
+	}
+
+	// Process in chunks of 8 bytes
+	remLen := len(buf) % 8
+	for len(buf) >= 8 {
+		q := crc ^ binary.LittleEndian.Uint32(buf[:4])
+		crc = doCRC8(q, tab)
+		q = binary.LittleEndian.Uint32(buf[4:8])
+		crc ^= doCRC4(q, tab)
+		buf = buf[8:]
+	}
+
+	// Process remaining bytes
+	for _, b := range buf[:remLen] {
+		crc = doCRC(crc, b, &tab[0])
+	}
+
+	return crc
+}