support lz4 compressed ramdisk

support lz4: Fix #32
other enhancements:
  avbtool: v1.1: for BootV2 before Android 11
  avbtool: v1.2: for BootV2(Android 11+), Vendor Boot, BootV3
  fix VB1.0 parsing error
  travis: osx xcode12.2, macOS 10.15.7, jdk 14.0.2

.travis.yml

@@ -3,10 +3,12 @@ os:
   - linux
   - osx
 dist: focal
+osx_image: xcode12.2
 addons:
   apt:
     packages:
       - cpio
+      - xz-utils
       - libblkid-dev
       - liblz4-tool
       - device-tree-compiler

README.md

@@ -11,7 +11,7 @@ A tool for reverse engineering Android ROM images.  (working on ![Linux](doc/lin
 * install required packages
 
   ```bash
-  sudo apt install device-tree-compiler lz4 zlib1g-dev cpio
+  sudo apt install device-tree-compiler lz4 xz zlib1g-dev cpio
   ```
 
 * get the tool

bbootimg/build.gradle.kts

@@ -22,6 +22,7 @@ dependencies {
     implementation("com.google.guava:guava:18.0")
     implementation("org.apache.commons:commons-exec:1.3")
     implementation("org.apache.commons:commons-compress:1.16.1")
+    implementation("org.tukaani:xz:1.8")
     implementation("commons-codec:commons-codec:1.11")
     implementation("junit:junit:4.12")
     implementation("org.bouncycastle:bcprov-jdk15on:1.57")

bbootimg/src/main/kotlin/Helper.kt

@@ -5,6 +5,7 @@ import com.google.common.math.BigIntegerMath
 import org.apache.commons.codec.binary.Hex
 import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream
 import org.apache.commons.compress.compressors.gzip.GzipParameters
+import org.apache.commons.compress.compressors.xz.XZCompressorInputStream
 import org.apache.commons.exec.CommandLine
 import org.apache.commons.exec.DefaultExecutor
 import org.apache.commons.exec.ExecuteException
@@ -18,6 +19,7 @@ import java.nio.file.Paths
 import java.util.*
 import java.util.zip.GZIPInputStream
 import java.util.zip.GZIPOutputStream
+import java.util.zip.ZipException
 import javax.crypto.Cipher
 
 @OptIn(ExperimentalUnsignedTypes::class)
@@ -89,6 +91,54 @@ class Helper {
             return data
         }
 
+        fun isGZ(compressedFile: String): Boolean {
+            return try {
+                GZIPInputStream(FileInputStream(compressedFile)).use { }
+                true
+            } catch (e: ZipException) {
+                false
+            }
+        }
+
+        fun isXZ(compressedFile: String): Boolean {
+            return try {
+                XZCompressorInputStream(FileInputStream(compressedFile)).use { }
+                true
+            } catch (e: ZipException) {
+                false
+            }
+        }
+
+        fun isLZ4(compressedFile: String): Boolean {
+            return try {
+                "lz4 -t $compressedFile".check_call()
+                true
+            } catch (e: Exception) {
+                false
+            }
+        }
+
+        fun decompressLZ4(lz4File: String, outFile: String) {
+            "lz4 -d -fv $lz4File $outFile".check_call()
+        }
+
+        fun compressLZ4(lz4File: String, inputStream: InputStream) {
+            val fos = FileOutputStream(File(lz4File))
+            val baosE = ByteArrayOutputStream()
+            DefaultExecutor().let { exec ->
+                exec.streamHandler = PumpStreamHandler(fos, baosE, inputStream)
+                val cmd = CommandLine.parse("lz4 -l -12 --favor-decSpeed")
+                log.info(cmd.toString())
+                exec.execute(cmd)
+            }
+            baosE.toByteArray().let {
+                if (it.isNotEmpty()) {
+                    log.warn(String(it))
+                }
+            }
+            fos.close()
+        }
+
         @Throws(IOException::class)
         fun gnuZipFile(compressedFile: String, decompressedFile: String) {
             val buffer = ByteArray(1024)

bbootimg/src/main/kotlin/bootimg/Common.kt

@@ -9,15 +9,13 @@ import org.apache.commons.exec.CommandLine
 import org.apache.commons.exec.DefaultExecutor
 import org.apache.commons.exec.PumpStreamHandler
 import org.slf4j.LoggerFactory
-import java.io.ByteArrayInputStream
-import java.io.ByteArrayOutputStream
-import java.io.File
-import java.io.FileInputStream
+import java.io.*
 import java.nio.ByteBuffer
 import java.nio.ByteOrder
 import java.security.MessageDigest
 import java.util.regex.Pattern
 
+
 @OptIn(ExperimentalUnsignedTypes::class)
 class Common {
     data class VeritySignature(
@@ -108,10 +106,25 @@ class Common {
             parseKernelInfo(s.dumpFile)
         }
 
-        fun dumpRamdisk(s: Slice, root: String) {
+        fun dumpRamdisk(s: Slice, root: String): String {
+            var ret = "gz"
             Helper.extractFile(s.srcFile, s.dumpFile, s.offset.toLong(), s.length)
-            Helper.unGnuzipFile(s.dumpFile, s.dumpFile.removeSuffix(".gz"))
+            when {
+                Helper.isGZ(s.dumpFile) -> {
+                    Helper.unGnuzipFile(s.dumpFile, s.dumpFile.removeSuffix(".gz"))
+                }
+                Helper.isLZ4(s.dumpFile) -> {
+                    log.info("ramdisk is compressed lz4")
+                    Helper.decompressLZ4(s.dumpFile, s.dumpFile.removeSuffix(".gz"))
+                    File(s.dumpFile).renameTo(File(s.dumpFile.replace(".gz", ".lz4")))
+                    ret = "lz4"
+                }
+                else -> {
+                    throw IllegalArgumentException("ramdisk is in unknown format")
+                }
+            }
             unpackRamdisk(s.dumpFile.removeSuffix(".gz"), root)
+            return ret
         }
 
         fun dumpDtb(s: Slice) {
@@ -185,7 +198,17 @@ class Common {
                 log.info("CMD: $cmdline -> PIPE -> $ramdiskGz")
                 exec.execute(CommandLine.parse(cmdline))
             }
-            Helper.gnuZipFile2(ramdiskGz, ByteArrayInputStream(outputStream.toByteArray()))
+            when {
+                ramdiskGz.endsWith(".gz") -> {
+                    Helper.gnuZipFile2(ramdiskGz, ByteArrayInputStream(outputStream.toByteArray()))
+                }
+                ramdiskGz.endsWith(".lz4") -> {
+                    Helper.compressLZ4(ramdiskGz, ByteArrayInputStream(outputStream.toByteArray()))
+                }
+                else -> {
+                    throw IllegalArgumentException("$ramdiskGz is not supported")
+                }
+            }
             log.info("$ramdiskGz is ready")
         }
 

bbootimg/src/main/kotlin/bootimg/Signer.kt

@@ -16,8 +16,7 @@ class Signer {
     companion object {
         private val log = LoggerFactory.getLogger(Signer::class.java)
 
-        fun signAVB(output: String, imageSize: Long) {
-            val avbtool = Helper.prop("avbtool")
+        fun signAVB(output: String, imageSize: Long, avbtool: String) {
             log.info("Adding hash_footer with verified-boot 2.0 style")
             val ai = ObjectMapper().readValue(File(getJsonFileName(output)), AVBInfo::class.java)
             val alg = Algorithms.get(ai.header!!.algorithm_type.toInt())
@@ -38,6 +37,7 @@ class Signer {
                 addArguments("--partition_name ${bootDesc.partition_name}")
                 addArguments("--hash_algorithm ${bootDesc.hash_algorithm}")
                 addArguments("--algorithm ${alg!!.name}")
+                addArguments("--rollback_index ${ai.header!!.rollback_index}")
                 if (alg.defaultKey.isNotBlank()) {
                     addArguments("--key ${alg.defaultKey}")
                 }

bbootimg/src/main/kotlin/bootimg/v2/BootV2.kt

@@ -78,7 +78,7 @@ data class BootV2(
                     theInfo.osPatchLevel = bh2.osPatchLevel
                     if (Avb.hasAvbFooter(fileName)) {
                         theInfo.verify = "VB2.0"
-                        Avb.verifyAVBIntegrity(fileName, Helper.prop("avbtool"))
+                        Avb.verifyAVBIntegrity(fileName, String.format(Helper.prop("avbtool"), "v1.2"))
                     } else {
                         theInfo.verify = "VB1.0"
                     }
@@ -194,10 +194,14 @@ data class BootV2(
     }
 
     fun extractVBMeta(): BootV2 {
-        Avb().parseVbMeta(info.output)
-        if (File("vbmeta.img").exists()) {
-            log.warn("Found vbmeta.img, parsing ...")
-            VBMetaParser().unpack("vbmeta.img")
+        if (this.info.verify == "VB2.0") {
+            Avb().parseVbMeta(info.output)
+            if (File("vbmeta.img").exists()) {
+                log.warn("Found vbmeta.img, parsing ...")
+                VBMetaParser().unpack("vbmeta.img")
+            }
+        } else {
+            log.info("verify type is ${this.info.verify}, skip AVB parsing")
         }
         return this
     }
@@ -469,8 +473,9 @@ data class BootV2(
     }
 
     fun sign(): BootV2 {
+        val avbtool = String.format(Helper.prop("avbtool"), if (parseOsMajor() > 10) "v1.2" else "v1.1")
         if (info.verify == "VB2.0") {
-            Signer.signAVB(info.output, this.info.imageSize)
+            Signer.signAVB(info.output, this.info.imageSize, avbtool)
             log.info("Adding hash_footer with verified-boot 2.0 style")
         } else {
             Signer.signVB1(info.output + ".clear", info.output + ".signed")

bbootimg/src/main/kotlin/bootimg/v3/BootV3.kt

@@ -87,7 +87,7 @@ data class BootV3(var info: MiscInfo = MiscInfo(),
             log.warn("Use prebuilt ramdisk file: ${this.ramdisk.file}")
         } else {
             File(this.ramdisk.file).deleleIfExists()
-            File(this.ramdisk.file.removeSuffix(".gz")).deleleIfExists()
+            File(this.ramdisk.file.replaceFirst("[.][^.]+$", "")).deleleIfExists()
             C.packRootfs("$workDir/root", this.ramdisk.file, parseOsMajor())
         }
         this.kernel.size = File(this.kernel.file).length().toInt()
@@ -124,7 +124,8 @@ data class BootV3(var info: MiscInfo = MiscInfo(),
     }
 
     fun sign(fileName: String): BootV3 {
-        Signer.signAVB(fileName, this.info.imageSize)
+        val avbtool = String.format(Helper.prop("avbtool"), "v1.2")
+        Signer.signAVB(fileName, this.info.imageSize, avbtool)
         return this
     }
 
@@ -146,7 +147,14 @@ data class BootV3(var info: MiscInfo = MiscInfo(),
         //kernel
         C.dumpKernel(C.Slice(info.output, kernel.position, kernel.size, kernel.file))
         //ramdisk
-        C.dumpRamdisk(C.Slice(info.output, ramdisk.position, ramdisk.size, ramdisk.file), "${workDir}root")
+        val fmt = C.dumpRamdisk(C.Slice(info.output, ramdisk.position, ramdisk.size, ramdisk.file), "${workDir}root")
+        if (fmt in listOf("xz", "lzma", "bz2", "lz4")) {
+            this.ramdisk.file = this.ramdisk.file.replace(".gz", ".$fmt")
+            //dump info again
+            ObjectMapper().writerWithDefaultPrettyPrinter().writeValue(File(workDir + this.info.json), this)
+        } else {
+            throw IllegalArgumentException("unknown format $fmt")
+        }
         return this
     }
 

bbootimg/src/main/kotlin/bootimg/v3/VendorBoot.kt

@@ -121,7 +121,8 @@ data class VendorBoot(var info: MiscInfo = MiscInfo(),
     }
 
     fun sign(): VendorBoot {
-        Signer.signAVB(info.output, this.info.imageSize)
+        val avbtool = String.format(Helper.prop("avbtool"), "v1.2")
+        Signer.signAVB(info.output, this.info.imageSize, avbtool)
         return this
     }
 

bbootimg/src/main/resources/general.cfg

@@ -1,6 +1,6 @@
 workDir = build/unzip_boot/
 mkbootfsBin = aosp/mkbootfs.%d/build/exe/mkbootfs/mkbootfs
-avbtool = aosp/avb/avbtool
+avbtool = aosp/avb/avbtool.%s.py
 bootSigner = aosp/boot_signer/build/libs/boot_signer.jar
 verity_pk8 = aosp/security/verity.pk8
 verity_pem = aosp/security/verity.x509.pem