From 7815b2fafa1897555f56bffc8a68fa8e94455ddb Mon Sep 17 00:00:00 2001 From: Lasse Collin Date: Sun, 21 Jul 2024 16:36:16 +0300 Subject: [PATCH 01/14] MAINTAINERS: add XZ Embedded maintainer mainline inclusion from mainline-v6.12-rc1 category: feature Patch series "xz: Updates to license, filters, and compression options", v2. XZ Embedded, the upstream project, switched from public domain to the BSD Zero Clause License (0BSD). Now matching SPDX license identifiers can be added. Documentation was revised. Fix syntax errors in kernel-doc comments in . The xz_dec API docs from are now included in Documentation/staging/xz.rst. The new ARM64 and RISC-V filters can be used for kernel decompression if CONFIG_EFI_ZBOOT=y. The filters can be used by Squashfs too. (Userspace Squashfs-tools already had the ARM64 filter support committed but it was reverted due to backdoor fears. I try to get ARM64 and RISC-V filter support added to Squashfs-tools somewhat soon.) Account for the default threading change made in the xz command line tool version 5.6.0. Tweak kernel compression options for archs that support XZ compressed kernel. This patch (of 16): I have been the maintainer of the upstream project since I submitted the code to Linux in 2010 but I forgot to add myself to MAINTAINERS. Link: https://lkml.kernel.org/r/20240721133633.47721-1-lasse.collin@tukaani.org Link: https://lkml.kernel.org/r/20240721133633.47721-2-lasse.collin@tukaani.org Signed-off-by: Lasse Collin Reviewed-by: Sam James Cc: Albert Ou Cc: Catalin Marinas Cc: Emil Renner Berthing Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Joel Stanley Cc: Jonathan Corbet Cc: Jubin Zhong Cc: Jules Maselbas Cc: Krzysztof Kozlowski Cc: Michael Ellerman Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Randy Dunlap Cc: Rui Li Cc: Simon Glass Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton (cherry picked from commit 22be8e6b1385d52e68b38599eadfa5d87966ab23) --- MAINTAINERS | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index ae46ea84ee5c..b28a310c28ec 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -23837,6 +23837,19 @@ S: Maintained F: drivers/spi/spi-xtensa-xtfpga.c F: sound/soc/xtensa/xtfpga-i2s.c +XZ EMBEDDED +M: Lasse Collin +S: Maintained +W: https://tukaani.org/xz/embedded.html +B: https://github.com/tukaani-project/xz-embedded/issues +C: irc://irc.libera.chat/tukaani +F: Documentation/staging/xz.rst +F: include/linux/decompress/unxz.h +F: include/linux/xz.h +F: lib/decompress_unxz.c +F: lib/xz/ +F: scripts/xz_wrap.sh + YAM DRIVER FOR AX.25 M: Jean-Paul Roubelat L: linux-hams@vger.kernel.org From 1df28dcbc1dc3ef95a70455fc589ab25c57f2ed1 Mon Sep 17 00:00:00 2001 From: Lasse Collin Date: Sun, 21 Jul 2024 16:36:17 +0300 Subject: [PATCH 02/14] LICENSES: add 0BSD license text mainline inclusion from mainline-v6.12-rc1 category: feature The license text was copied from: https://spdx.org/licenses/0BSD.html Link: https://lkml.kernel.org/r/20240721133633.47721-3-lasse.collin@tukaani.org Signed-off-by: Lasse Collin Reviewed-by: Sam James Cc: Thomas Gleixner Cc: Greg Kroah-Hartman Cc: Albert Ou Cc: Catalin Marinas Cc: Emil Renner Berthing Cc: Herbert Xu Cc: Joel Stanley Cc: Jonathan Corbet Cc: Jubin Zhong Cc: Jules Maselbas Cc: Krzysztof Kozlowski Cc: Michael Ellerman Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Randy Dunlap Cc: Rui Li Cc: Simon Glass Cc: Will Deacon Signed-off-by: Andrew Morton (cherry picked from commit c1ccbbaa76c989aeaecaefee1a3d890cc674b005) --- LICENSES/deprecated/0BSD | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 LICENSES/deprecated/0BSD diff --git a/LICENSES/deprecated/0BSD b/LICENSES/deprecated/0BSD new file mode 100644 index 000000000000..e4b95b749966 --- /dev/null +++ b/LICENSES/deprecated/0BSD @@ -0,0 +1,23 @@ +Valid-License-Identifier: 0BSD +SPDX-URL: https://spdx.org/licenses/0BSD.html +Usage-Guide: + To use the BSD Zero Clause License put the following SPDX tag/value + pair into a comment according to the placement guidelines in the + licensing rules documentation: + SPDX-License-Identifier: 0BSD +License-Text: + +BSD Zero Clause License + +Copyright (c) + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. From d71989f9483fc9f2f87f2102df9191de789cb56a Mon Sep 17 00:00:00 2001 From: Lasse Collin Date: Sun, 21 Jul 2024 16:36:18 +0300 Subject: [PATCH 03/14] xz: switch from public domain to BSD Zero Clause License (0BSD) mainline inclusion from mainline-v6.12-rc1 category: feature Remove the public domain notices and add SPDX license identifiers. Change MODULE_LICENSE from "GPL" to "Dual BSD/GPL" because 0BSD should count as a BSD license variant here. The switch to 0BSD was done in the upstream XZ Embedded project because public domain has (real or perceived) legal issues in some jurisdictions. Link: https://lkml.kernel.org/r/20240721133633.47721-4-lasse.collin@tukaani.org Signed-off-by: Lasse Collin Reviewed-by: Sam James Cc: Thomas Gleixner Cc: Greg Kroah-Hartman Cc: Albert Ou Cc: Catalin Marinas Cc: Emil Renner Berthing Cc: Herbert Xu Cc: Joel Stanley Cc: Jonathan Corbet Cc: Jubin Zhong Cc: Jules Maselbas Cc: Krzysztof Kozlowski Cc: Michael Ellerman Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Randy Dunlap Cc: Rui Li Cc: Simon Glass Cc: Will Deacon Signed-off-by: Andrew Morton (cherry picked from commit 836d13a6ef8a2eb0eab2bd2de06f2deabc62b060) --- include/linux/decompress/unxz.h | 5 ++--- include/linux/xz.h | 5 ++--- lib/decompress_unxz.c | 5 ++--- lib/xz/xz_crc32.c | 5 ++--- lib/xz/xz_dec_bcj.c | 5 ++--- lib/xz/xz_dec_lzma2.c | 5 ++--- lib/xz/xz_dec_stream.c | 5 ++--- lib/xz/xz_dec_syms.c | 12 +++--------- lib/xz/xz_dec_test.c | 12 +++--------- lib/xz/xz_lzma2.h | 5 ++--- lib/xz/xz_private.h | 5 ++--- lib/xz/xz_stream.h | 5 ++--- scripts/xz_wrap.sh | 5 +---- 13 files changed, 27 insertions(+), 52 deletions(-) diff --git a/include/linux/decompress/unxz.h b/include/linux/decompress/unxz.h index f764e2a7201e..3dd2658a9dab 100644 --- a/include/linux/decompress/unxz.h +++ b/include/linux/decompress/unxz.h @@ -1,10 +1,9 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Wrapper for decompressing XZ-compressed kernel, initramfs, and initrd * * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef DECOMPRESS_UNXZ_H diff --git a/include/linux/xz.h b/include/linux/xz.h index 7285ca5d56e9..5728d57aecc0 100644 --- a/include/linux/xz.h +++ b/include/linux/xz.h @@ -1,11 +1,10 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * XZ decompressor * * Authors: Lasse Collin * Igor Pavlov - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef XZ_H diff --git a/lib/decompress_unxz.c b/lib/decompress_unxz.c index 353268b9f129..e056dcc922da 100644 --- a/lib/decompress_unxz.c +++ b/lib/decompress_unxz.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: 0BSD + /* * Wrapper for decompressing XZ-compressed kernel, initramfs, and initrd * * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ /* diff --git a/lib/xz/xz_crc32.c b/lib/xz/xz_crc32.c index 5627b00fca29..effdf34ec48d 100644 --- a/lib/xz/xz_crc32.c +++ b/lib/xz/xz_crc32.c @@ -1,11 +1,10 @@ +// SPDX-License-Identifier: 0BSD + /* * CRC32 using the polynomial from IEEE-802.3 * * Authors: Lasse Collin * Igor Pavlov - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ /* diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c index ef449e97d1a1..ab9237ed6db8 100644 --- a/lib/xz/xz_dec_bcj.c +++ b/lib/xz/xz_dec_bcj.c @@ -1,11 +1,10 @@ +// SPDX-License-Identifier: 0BSD + /* * Branch/Call/Jump (BCJ) filter decoders * * Authors: Lasse Collin * Igor Pavlov - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include "xz_private.h" diff --git a/lib/xz/xz_dec_lzma2.c b/lib/xz/xz_dec_lzma2.c index 27ce34520e78..613939f5dd6c 100644 --- a/lib/xz/xz_dec_lzma2.c +++ b/lib/xz/xz_dec_lzma2.c @@ -1,11 +1,10 @@ +// SPDX-License-Identifier: 0BSD + /* * LZMA2 decoder * * Authors: Lasse Collin * Igor Pavlov - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include "xz_private.h" diff --git a/lib/xz/xz_dec_stream.c b/lib/xz/xz_dec_stream.c index 683570b93a8c..0058406ccd17 100644 --- a/lib/xz/xz_dec_stream.c +++ b/lib/xz/xz_dec_stream.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: 0BSD + /* * .xz Stream decoder * * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include "xz_private.h" diff --git a/lib/xz/xz_dec_syms.c b/lib/xz/xz_dec_syms.c index 61098c67a413..495d2cc2e6e8 100644 --- a/lib/xz/xz_dec_syms.c +++ b/lib/xz/xz_dec_syms.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: 0BSD + /* * XZ decoder module information * * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include @@ -25,9 +24,4 @@ EXPORT_SYMBOL(xz_dec_microlzma_end); MODULE_DESCRIPTION("XZ decompressor"); MODULE_VERSION("1.1"); MODULE_AUTHOR("Lasse Collin and Igor Pavlov"); - -/* - * This code is in the public domain, but in Linux it's simplest to just - * say it's GPL and consider the authors as the copyright holders. - */ -MODULE_LICENSE("GPL"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/lib/xz/xz_dec_test.c b/lib/xz/xz_dec_test.c index da28a19d6c98..53d3600f2ddb 100644 --- a/lib/xz/xz_dec_test.c +++ b/lib/xz/xz_dec_test.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: 0BSD + /* * XZ decoder tester * * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include @@ -212,9 +211,4 @@ module_exit(xz_dec_test_exit); MODULE_DESCRIPTION("XZ decompressor tester"); MODULE_VERSION("1.0"); MODULE_AUTHOR("Lasse Collin "); - -/* - * This code is in the public domain, but in Linux it's simplest to just - * say it's GPL and consider the authors as the copyright holders. - */ -MODULE_LICENSE("GPL"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/lib/xz/xz_lzma2.h b/lib/xz/xz_lzma2.h index 92d852d4f87a..d2632b7dfb9c 100644 --- a/lib/xz/xz_lzma2.h +++ b/lib/xz/xz_lzma2.h @@ -1,11 +1,10 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * LZMA2 definitions * * Authors: Lasse Collin * Igor Pavlov - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef XZ_LZMA2_H diff --git a/lib/xz/xz_private.h b/lib/xz/xz_private.h index d9fd49b45fd7..811add814ae4 100644 --- a/lib/xz/xz_private.h +++ b/lib/xz/xz_private.h @@ -1,10 +1,9 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Private includes and definitions * * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef XZ_PRIVATE_H diff --git a/lib/xz/xz_stream.h b/lib/xz/xz_stream.h index 430bb3a0d195..55f9f6f94b78 100644 --- a/lib/xz/xz_stream.h +++ b/lib/xz/xz_stream.h @@ -1,10 +1,9 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Definitions for handling the .xz file format * * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef XZ_STREAM_H diff --git a/scripts/xz_wrap.sh b/scripts/xz_wrap.sh index 76e9cbcfbeab..c06ff99b0af7 100755 --- a/scripts/xz_wrap.sh +++ b/scripts/xz_wrap.sh @@ -1,13 +1,10 @@ #!/bin/sh +# SPDX-License-Identifier: 0BSD # # This is a wrapper for xz to compress the kernel image using appropriate # compression options depending on the architecture. # # Author: Lasse Collin -# -# This file has been put into the public domain. -# You can do whatever you want with this file. -# BCJ= LZMA2OPTS= From d59f6810c694f99a2c637a7cd66aae1ccd35d0bd Mon Sep 17 00:00:00 2001 From: Lasse Collin Date: Sun, 21 Jul 2024 16:36:19 +0300 Subject: [PATCH 04/14] xz: fix comments and coding style mainline inclusion from mainline-v6.12-rc1 category: feature - Fix comments that were no longer in sync with the code below them. - Fix language errors. - Fix coding style. Link: https://lkml.kernel.org/r/20240721133633.47721-5-lasse.collin@tukaani.org Signed-off-by: Lasse Collin Reviewed-by: Sam James Cc: Albert Ou Cc: Catalin Marinas Cc: Emil Renner Berthing Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Joel Stanley Cc: Jonathan Corbet Cc: Jubin Zhong Cc: Jules Maselbas Cc: Krzysztof Kozlowski Cc: Michael Ellerman Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Randy Dunlap Cc: Rui Li Cc: Simon Glass Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton (cherry picked from commit ff221153aafa08159f3dcc187c6f3a7a837e1c3d) --- lib/decompress_unxz.c | 20 ++++++++++---------- lib/xz/Kconfig | 3 ++- scripts/Makefile.lib | 13 ++++++++----- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/lib/decompress_unxz.c b/lib/decompress_unxz.c index e056dcc922da..86a07f2af601 100644 --- a/lib/decompress_unxz.c +++ b/lib/decompress_unxz.c @@ -102,7 +102,7 @@ #ifdef STATIC # define XZ_PREBOOT #else -#include +# include #endif #ifdef __KERNEL__ # include @@ -222,7 +222,7 @@ void *memmove(void *dest, const void *src, size_t size) #endif /* - * Since we need memmove anyway, would use it as memcpy too. + * Since we need memmove anyway, we could use it as memcpy too. * Commented out for now to avoid breaking things. */ /* @@ -392,17 +392,17 @@ STATIC int INIT unxz(unsigned char *in, long in_size, } /* - * This macro is used by architecture-specific files to decompress + * This function is used by architecture-specific files to decompress * the kernel image. */ #ifdef XZ_PREBOOT -STATIC int INIT __decompress(unsigned char *buf, long len, - long (*fill)(void*, unsigned long), - long (*flush)(void*, unsigned long), - unsigned char *out_buf, long olen, - long *pos, - void (*error)(char *x)) +STATIC int INIT __decompress(unsigned char *in, long in_size, + long (*fill)(void *dest, unsigned long size), + long (*flush)(void *src, unsigned long size), + unsigned char *out, long out_size, + long *in_used, + void (*error)(char *x)) { - return unxz(buf, len, fill, flush, out_buf, pos, error); + return unxz(in, in_size, fill, flush, out, in_used, error); } #endif diff --git a/lib/xz/Kconfig b/lib/xz/Kconfig index adce22ac18d6..45a96cf6043b 100644 --- a/lib/xz/Kconfig +++ b/lib/xz/Kconfig @@ -5,7 +5,8 @@ config XZ_DEC help LZMA2 compression algorithm and BCJ filters are supported using the .xz file format as the container. For integrity checking, - CRC32 is supported. See Documentation/staging/xz.rst for more information. + CRC32 is supported. See Documentation/staging/xz.rst for more + information. if XZ_DEC diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index e702552fb131..7da88020acb4 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -502,14 +502,17 @@ quiet_cmd_uimage = UIMAGE $@ # XZ # --------------------------------------------------------------------------- -# Use xzkern to compress the kernel image and xzmisc to compress other things. +# Use xzkern or xzkern_with_size to compress the kernel image and xzmisc to +# compress other things. # # xzkern uses a big LZMA2 dictionary since it doesn't increase memory usage # of the kernel decompressor. A BCJ filter is used if it is available for -# the target architecture. xzkern also appends uncompressed size of the data -# using size_append. The .xz format has the size information available at -# the end of the file too, but it's in more complex format and it's good to -# avoid changing the part of the boot code that reads the uncompressed size. +# the target architecture. +# +# xzkern_with_size also appends uncompressed size of the data using +# size_append. The .xz format has the size information available at the end +# of the file too, but it's in more complex format and it's good to avoid +# changing the part of the boot code that reads the uncompressed size. # Note that the bytes added by size_append will make the xz tool think that # the file is corrupt. This is expected. # From 8aedfb62295f076f6aa2e021676d20ddd3c98414 Mon Sep 17 00:00:00 2001 From: Lasse Collin Date: Sun, 21 Jul 2024 16:36:20 +0300 Subject: [PATCH 05/14] xz: fix kernel-doc formatting errors in xz.h mainline inclusion from mainline-v6.12-rc1 category: feature The opaque structs xz_dec and xz_dec_microlzma are declared in xz.h but their definitions are in xz_dec_lzma2.c without kernel-doc comments. Use regular comments for these structs in xz.h to avoid errors when building the docs. Add a few missing colons. Link: https://lkml.kernel.org/r/20240721133633.47721-6-lasse.collin@tukaani.org Signed-off-by: Lasse Collin Reviewed-by: Sam James Cc: Albert Ou Cc: Catalin Marinas Cc: Emil Renner Berthing Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Joel Stanley Cc: Jonathan Corbet Cc: Jubin Zhong Cc: Jules Maselbas Cc: Krzysztof Kozlowski Cc: Michael Ellerman Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Randy Dunlap Cc: Rui Li Cc: Simon Glass Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton (cherry picked from commit ad8c67b870d108aa1286f4fc76d0c29a736fd75e) --- include/linux/xz.h | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/include/linux/xz.h b/include/linux/xz.h index 5728d57aecc0..af1e075d9add 100644 --- a/include/linux/xz.h +++ b/include/linux/xz.h @@ -142,7 +142,7 @@ struct xz_buf { size_t out_size; }; -/** +/* * struct xz_dec - Opaque type to hold the XZ decoder state */ struct xz_dec; @@ -240,15 +240,16 @@ XZ_EXTERN void xz_dec_end(struct xz_dec *s); * marked with XZ_EXTERN. This avoids warnings about static functions that * are never defined. */ -/** + +/* * struct xz_dec_microlzma - Opaque type to hold the MicroLZMA decoder state */ struct xz_dec_microlzma; /** * xz_dec_microlzma_alloc() - Allocate memory for the MicroLZMA decoder - * @mode XZ_SINGLE or XZ_PREALLOC - * @dict_size LZMA dictionary size. This must be at least 4 KiB and + * @mode: XZ_SINGLE or XZ_PREALLOC + * @dict_size: LZMA dictionary size. This must be at least 4 KiB and * at most 3 GiB. * * In contrast to xz_dec_init(), this function only allocates the memory @@ -276,15 +277,15 @@ extern struct xz_dec_microlzma *xz_dec_microlzma_alloc(enum xz_mode mode, /** * xz_dec_microlzma_reset() - Reset the MicroLZMA decoder state - * @s Decoder state allocated using xz_dec_microlzma_alloc() - * @comp_size Compressed size of the input stream - * @uncomp_size Uncompressed size of the input stream. A value smaller + * @s: Decoder state allocated using xz_dec_microlzma_alloc() + * @comp_size: Compressed size of the input stream + * @uncomp_size: Uncompressed size of the input stream. A value smaller * than the real uncompressed size of the input stream can * be specified if uncomp_size_is_exact is set to false. * uncomp_size can never be set to a value larger than the * expected real uncompressed size because it would eventually * result in XZ_DATA_ERROR. - * @uncomp_size_is_exact This is an int instead of bool to avoid + * @uncomp_size_is_exact: This is an int instead of bool to avoid * requiring stdbool.h. This should normally be set to true. * When this is set to false, error detection is weaker. */ @@ -294,7 +295,7 @@ extern void xz_dec_microlzma_reset(struct xz_dec_microlzma *s, /** * xz_dec_microlzma_run() - Run the MicroLZMA decoder - * @s Decoder state initialized using xz_dec_microlzma_reset() + * @s: Decoder state initialized using xz_dec_microlzma_reset() * @b: Input and output buffers * * This works similarly to xz_dec_run() with a few important differences. From 93d5c326dfda7960b2f3a769515c678f66e610fc Mon Sep 17 00:00:00 2001 From: Lasse Collin Date: Sun, 21 Jul 2024 16:36:21 +0300 Subject: [PATCH 06/14] xz: improve the MicroLZMA kernel-doc in xz.h mainline inclusion from mainline-v6.12-rc1 category: feature Move the description of the format into a "DOC:" comment. Emphasize that MicroLZMA functions aren't usually needed. Link: https://lkml.kernel.org/r/20240721133633.47721-7-lasse.collin@tukaani.org Signed-off-by: Lasse Collin Reviewed-by: Sam James Cc: Albert Ou Cc: Catalin Marinas Cc: Emil Renner Berthing Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Joel Stanley Cc: Jonathan Corbet Cc: Jubin Zhong Cc: Jules Maselbas Cc: Krzysztof Kozlowski Cc: Michael Ellerman Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Randy Dunlap Cc: Rui Li Cc: Simon Glass Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton (cherry picked from commit 0f2c5996340b69d167d3f6ca38d3012204787ac1) --- include/linux/xz.h | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/include/linux/xz.h b/include/linux/xz.h index af1e075d9add..701d62c02b9a 100644 --- a/include/linux/xz.h +++ b/include/linux/xz.h @@ -232,9 +232,18 @@ XZ_EXTERN void xz_dec_reset(struct xz_dec *s); */ XZ_EXTERN void xz_dec_end(struct xz_dec *s); -/* - * Decompressor for MicroLZMA, an LZMA variant with a very minimal header. - * See xz_dec_microlzma_alloc() below for details. +/** + * DOC: MicroLZMA decompressor + * + * This MicroLZMA header format was created for use in EROFS but may be used + * by others too. **In most cases one needs the XZ APIs above instead.** + * + * The compressed format supported by this decoder is a raw LZMA stream + * whose first byte (always 0x00) has been replaced with bitwise-negation + * of the LZMA properties (lc/lp/pb) byte. For example, if lc/lp/pb is + * 3/0/2, the first byte is 0xA2. This way the first byte can never be 0x00. + * Just like with LZMA2, lc + lp <= 4 must be true. The LZMA end-of-stream + * marker must not be used. The unused values are reserved for future use. * * These functions aren't used or available in preboot code and thus aren't * marked with XZ_EXTERN. This avoids warnings about static functions that @@ -262,15 +271,6 @@ struct xz_dec_microlzma; * On success, xz_dec_microlzma_alloc() returns a pointer to * struct xz_dec_microlzma. If memory allocation fails or * dict_size is invalid, NULL is returned. - * - * The compressed format supported by this decoder is a raw LZMA stream - * whose first byte (always 0x00) has been replaced with bitwise-negation - * of the LZMA properties (lc/lp/pb) byte. For example, if lc/lp/pb is - * 3/0/2, the first byte is 0xA2. This way the first byte can never be 0x00. - * Just like with LZMA2, lc + lp <= 4 must be true. The LZMA end-of-stream - * marker must not be used. The unused values are reserved for future use. - * This MicroLZMA header format was created for use in EROFS but may be used - * by others too. */ extern struct xz_dec_microlzma *xz_dec_microlzma_alloc(enum xz_mode mode, uint32_t dict_size); From 3bd77a38d92119cf63df9e8e58b82e909cb2df4d Mon Sep 17 00:00:00 2001 From: Lasse Collin Date: Sun, 21 Jul 2024 16:36:22 +0300 Subject: [PATCH 07/14] xz: Documentation/staging/xz.rst: Revise thoroughly mainline inclusion from mainline-v6.12-rc1 category: feature Add SPDX license identifier. Omit xz_dec_test info. That isn't relevant to developers of non-XZ code. Revise the docs about xzkern and add xzkern_with_size. The latter was added to scripts/Makefile.lib in the commit 7ce7e984ab2b ("kbuild: rename cmd_{bzip2,lzma,lzo,lz4,xzkern,zstd22}"). Omit contact info as MAINTAINERS has it. Omit other info that is outdated or not relevant in the kernel context. Include the xz_dec kernel-doc from include/linux/xz.h. Link: https://lkml.kernel.org/r/20240721133633.47721-8-lasse.collin@tukaani.org Signed-off-by: Lasse Collin Reviewed-by: Sam James Reviewed-by: Randy Dunlap Tested-by: Randy Dunlap Cc: Rui Li Cc: Albert Ou Cc: Catalin Marinas Cc: Emil Renner Berthing Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Joel Stanley Cc: Jonathan Corbet Cc: Jubin Zhong Cc: Jules Maselbas Cc: Krzysztof Kozlowski Cc: Michael Ellerman Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Simon Glass Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton (cherry picked from commit 64167246791eb38af4cbe8bc93fc2701c71fd17e) --- Documentation/staging/xz.rst | 160 +++++++++++++++-------------------- 1 file changed, 67 insertions(+), 93 deletions(-) diff --git a/Documentation/staging/xz.rst b/Documentation/staging/xz.rst index b2f5ff12a161..e1054e9a8e65 100644 --- a/Documentation/staging/xz.rst +++ b/Documentation/staging/xz.rst @@ -1,3 +1,5 @@ +.. SPDX-License-Identifier: 0BSD + ============================ XZ data compression in Linux ============================ @@ -6,62 +8,55 @@ Introduction ============ XZ is a general purpose data compression format with high compression -ratio and relatively fast decompression. The primary compression -algorithm (filter) is LZMA2. Additional filters can be used to improve -compression ratio even further. E.g. Branch/Call/Jump (BCJ) filters -improve compression ratio of executable data. - -The XZ decompressor in Linux is called XZ Embedded. It supports -the LZMA2 filter and optionally also BCJ filters. CRC32 is supported -for integrity checking. The home page of XZ Embedded is at -, where you can find the -latest version and also information about using the code outside -the Linux kernel. - -For userspace, XZ Utils provide a zlib-like compression library -and a gzip-like command line tool. XZ Utils can be downloaded from -. +ratio. The XZ decompressor in Linux is called XZ Embedded. It supports +the LZMA2 filter and optionally also Branch/Call/Jump (BCJ) filters +for executable code. CRC32 is supported for integrity checking. + +See the `XZ Embedded`_ home page for the latest version which includes +a few optional extra features that aren't required in the Linux kernel +and information about using the code outside the Linux kernel. + +For userspace, `XZ Utils`_ provide a zlib-like compression library +and a gzip-like command line tool. + +.. _XZ Embedded: https://tukaani.org/xz/embedded.html +.. _XZ Utils: https://tukaani.org/xz/ XZ related components in the kernel =================================== The xz_dec module provides XZ decompressor with single-call (buffer -to buffer) and multi-call (stateful) APIs. The usage of the xz_dec -module is documented in include/linux/xz.h. - -The xz_dec_test module is for testing xz_dec. xz_dec_test is not -useful unless you are hacking the XZ decompressor. xz_dec_test -allocates a char device major dynamically to which one can write -.xz files from userspace. The decompressed output is thrown away. -Keep an eye on dmesg to see diagnostics printed by xz_dec_test. -See the xz_dec_test source code for the details. +to buffer) and multi-call (stateful) APIs in include/linux/xz.h. For decompressing the kernel image, initramfs, and initrd, there is a wrapper function in lib/decompress_unxz.c. Its API is the same as in other decompress_*.c files, which is defined in include/linux/decompress/generic.h. -scripts/xz_wrap.sh is a wrapper for the xz command line tool found -from XZ Utils. The wrapper sets compression options to values suitable -for compressing the kernel image. +For kernel makefiles, three commands are provided for use with +``$(call if_changed)``. They require the xz tool from XZ Utils. + +- ``$(call if_changed,xzkern)`` is for compressing the kernel image. + It runs the script scripts/xz_wrap.sh which uses arch-optimized + options and a big LZMA2 dictionary. -For kernel makefiles, two commands are provided for use with -$(call if_needed). The kernel image should be compressed with -$(call if_needed,xzkern) which will use a BCJ filter and a big LZMA2 -dictionary. It will also append a four-byte trailer containing the -uncompressed size of the file, which is needed by the boot code. -Other things should be compressed with $(call if_needed,xzmisc) -which will use no BCJ filter and 1 MiB LZMA2 dictionary. +- ``$(call if_changed,xzkern_with_size)`` is like ``xzkern`` above but + this also appends a four-byte trailer containing the uncompressed size + of the file. The trailer is needed by the boot code on some archs. + +- Other things can be compressed with ``$(call if_needed,xzmisc)`` + which will use no BCJ filter and 1 MiB LZMA2 dictionary. Notes on compression options ============================ -Since the XZ Embedded supports only streams with no integrity check or -CRC32, make sure that you don't use some other integrity check type -when encoding files that are supposed to be decoded by the kernel. With -liblzma, you need to use either LZMA_CHECK_NONE or LZMA_CHECK_CRC32 -when encoding. With the xz command line tool, use --check=none or ---check=crc32. +Since the XZ Embedded supports only streams with CRC32 or no integrity +check, make sure that you don't use some other integrity check type +when encoding files that are supposed to be decoded by the kernel. +With liblzma from XZ Utils, you need to use either ``LZMA_CHECK_CRC32`` +or ``LZMA_CHECK_NONE`` when encoding. With the ``xz`` command line tool, +use ``--check=crc32`` or ``--check=none`` to override the default +``--check=crc64``. Using CRC32 is strongly recommended unless there is some other layer which will verify the integrity of the uncompressed data anyway. @@ -71,57 +66,36 @@ by the decoder; you can only change the integrity check type (or disable it) for the actual uncompressed data. In userspace, LZMA2 is typically used with dictionary sizes of several -megabytes. The decoder needs to have the dictionary in RAM, thus big -dictionaries cannot be used for files that are intended to be decoded -by the kernel. 1 MiB is probably the maximum reasonable dictionary -size for in-kernel use (maybe more is OK for initramfs). The presets -in XZ Utils may not be optimal when creating files for the kernel, -so don't hesitate to use custom settings. Example:: - - xz --check=crc32 --lzma2=dict=512KiB inputfile - -An exception to above dictionary size limitation is when the decoder -is used in single-call mode. Decompressing the kernel itself is an -example of this situation. In single-call mode, the memory usage -doesn't depend on the dictionary size, and it is perfectly fine to -use a big dictionary: for maximum compression, the dictionary should -be at least as big as the uncompressed data itself. - -Future plans -============ +megabytes. The decoder needs to have the dictionary in RAM: + +- In multi-call mode the dictionary is allocated as part of the + decoder state. The reasonable maximum dictionary size for in-kernel + use will depend on the target hardware: a few megabytes is fine for + desktop systems while 64 KiB to 1 MiB might be more appropriate on + some embedded systems. + +- In single-call mode the output buffer is used as the dictionary + buffer. That is, the size of the dictionary doesn't affect the + decompressor memory usage at all. Only the base data structures + are allocated which take a little less than 30 KiB of memory. + For the best compression, the dictionary should be at least + as big as the uncompressed data. A notable example of single-call + mode is decompressing the kernel itself (except on PowerPC). + +The compression presets in XZ Utils may not be optimal when creating +files for the kernel, so don't hesitate to use custom settings to, +for example, set the dictionary size. Also, xz may produce a smaller +file in single-threaded mode so setting that explicitly is recommended. +Example:: + + xz --threads=1 --check=crc32 --lzma2=dict=512KiB inputfile + +xz_dec API +========== + +This is available with ``#include ``. + +``XZ_EXTERN`` is a macro used in the preboot code. Ignore it when +reading this documentation. -Creating a limited XZ encoder may be considered if people think it is -useful. LZMA2 is slower to compress than e.g. Deflate or LZO even at -the fastest settings, so it isn't clear if LZMA2 encoder is wanted -into the kernel. - -Support for limited random-access reading is planned for the -decompression code. I don't know if it could have any use in the -kernel, but I know that it would be useful in some embedded projects -outside the Linux kernel. - -Conformance to the .xz file format specification -================================================ - -There are a couple of corner cases where things have been simplified -at expense of detecting errors as early as possible. These should not -matter in practice all, since they don't cause security issues. But -it is good to know this if testing the code e.g. with the test files -from XZ Utils. - -Reporting bugs -============== - -Before reporting a bug, please check that it's not fixed already -at upstream. See to get the -latest code. - -Report bugs to or visit #tukaani on -Freenode and talk to Larhzu. I don't actively read LKML or other -kernel-related mailing lists, so if there's something I should know, -you should email to me personally or use IRC. - -Don't bother Igor Pavlov with questions about the XZ implementation -in the kernel or about XZ Utils. While these two implementations -include essential code that is directly based on Igor Pavlov's code, -these implementations aren't maintained nor supported by him. +.. kernel-doc:: include/linux/xz.h From 23be6003e1669cd165b79fd640c985451a16fb3c Mon Sep 17 00:00:00 2001 From: Lasse Collin Date: Sun, 21 Jul 2024 16:36:25 +0300 Subject: [PATCH 08/14] xz: optimize for-loop conditions in the BCJ decoders mainline inclusion from mainline-v6.12-rc1 category: feature Compilers cannot optimize the addition "i + 4" away since theoretically it could overflow. Link: https://lkml.kernel.org/r/20240721133633.47721-11-lasse.collin@tukaani.org Signed-off-by: Lasse Collin Reviewed-by: Sam James Cc: Albert Ou Cc: Catalin Marinas Cc: Emil Renner Berthing Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Joel Stanley Cc: Jonathan Corbet Cc: Jubin Zhong Cc: Jules Maselbas Cc: Krzysztof Kozlowski Cc: Michael Ellerman Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Randy Dunlap Cc: Rui Li Cc: Simon Glass Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton (cherry picked from commit bdfc0411717d52b9d2f00e48c452a61389814693) --- lib/xz/xz_dec_bcj.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c index ab9237ed6db8..e0b4bf4999c0 100644 --- a/lib/xz/xz_dec_bcj.c +++ b/lib/xz/xz_dec_bcj.c @@ -161,7 +161,9 @@ static size_t bcj_powerpc(struct xz_dec_bcj *s, uint8_t *buf, size_t size) size_t i; uint32_t instr; - for (i = 0; i + 4 <= size; i += 4) { + size &= ~(size_t)3; + + for (i = 0; i < size; i += 4) { instr = get_unaligned_be32(buf + i); if ((instr & 0xFC000003) == 0x48000001) { instr &= 0x03FFFFFC; @@ -218,7 +220,9 @@ static size_t bcj_ia64(struct xz_dec_bcj *s, uint8_t *buf, size_t size) /* Instruction normalized with bit_res for easier manipulation */ uint64_t norm; - for (i = 0; i + 16 <= size; i += 16) { + size &= ~(size_t)15; + + for (i = 0; i < size; i += 16) { mask = branch_table[buf[i] & 0x1F]; for (slot = 0, bit_pos = 5; slot < 3; ++slot, bit_pos += 41) { if (((mask >> slot) & 1) == 0) @@ -266,7 +270,9 @@ static size_t bcj_arm(struct xz_dec_bcj *s, uint8_t *buf, size_t size) size_t i; uint32_t addr; - for (i = 0; i + 4 <= size; i += 4) { + size &= ~(size_t)3; + + for (i = 0; i < size; i += 4) { if (buf[i + 3] == 0xEB) { addr = (uint32_t)buf[i] | ((uint32_t)buf[i + 1] << 8) | ((uint32_t)buf[i + 2] << 16); @@ -289,7 +295,12 @@ static size_t bcj_armthumb(struct xz_dec_bcj *s, uint8_t *buf, size_t size) size_t i; uint32_t addr; - for (i = 0; i + 4 <= size; i += 2) { + if (size < 4) + return 0; + + size -= 4; + + for (i = 0; i <= size; i += 2) { if ((buf[i + 1] & 0xF8) == 0xF0 && (buf[i + 3] & 0xF8) == 0xF8) { addr = (((uint32_t)buf[i + 1] & 0x07) << 19) @@ -317,7 +328,9 @@ static size_t bcj_sparc(struct xz_dec_bcj *s, uint8_t *buf, size_t size) size_t i; uint32_t instr; - for (i = 0; i + 4 <= size; i += 4) { + size &= ~(size_t)3; + + for (i = 0; i < size; i += 4) { instr = get_unaligned_be32(buf + i); if ((instr >> 22) == 0x100 || (instr >> 22) == 0x1FF) { instr <<= 2; From d597ba7d509832bfc41fa3988a2db7d8c47919b2 Mon Sep 17 00:00:00 2001 From: Lasse Collin Date: Sun, 21 Jul 2024 16:36:26 +0300 Subject: [PATCH 09/14] xz: Add ARM64 BCJ filter mainline inclusion from mainline-v6.12-rc1 category: feature Also omit a duplicated check for XZ_DEC_ARM in xz_private.h. A later commit updates lib/decompress_unxz.c to enable this filter for kernel decompression. lib/decompress_unxz.c is already used if CONFIG_EFI_ZBOOT=y && CONFIG_KERNEL_XZ=y. This filter can be used by Squashfs without modifications to the Squashfs kernel code (only needs support in userspace Squashfs-tools). Link: https://lkml.kernel.org/r/20240721133633.47721-12-lasse.collin@tukaani.org Signed-off-by: Lasse Collin Reviewed-by: Sam James Cc: Albert Ou Cc: Catalin Marinas Cc: Emil Renner Berthing Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Joel Stanley Cc: Jonathan Corbet Cc: Jubin Zhong Cc: Jules Maselbas Cc: Krzysztof Kozlowski Cc: Michael Ellerman Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Randy Dunlap Cc: Rui Li Cc: Simon Glass Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton (cherry picked from commit 4b62813f5e7d44a33ebd74f03da041712c702bf0) --- lib/xz/Kconfig | 5 +++++ lib/xz/xz_dec_bcj.c | 52 ++++++++++++++++++++++++++++++++++++++++++++- lib/xz/xz_private.h | 7 ++++-- 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/lib/xz/Kconfig b/lib/xz/Kconfig index 45a96cf6043b..ac3fd920da7d 100644 --- a/lib/xz/Kconfig +++ b/lib/xz/Kconfig @@ -35,6 +35,11 @@ config XZ_DEC_ARMTHUMB default y select XZ_DEC_BCJ +config XZ_DEC_ARM64 + bool "ARM64 BCJ filter decoder" if EXPERT + default y + select XZ_DEC_BCJ + config XZ_DEC_SPARC bool "SPARC BCJ filter decoder" if EXPERT default y diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c index e0b4bf4999c0..941198a8a55b 100644 --- a/lib/xz/xz_dec_bcj.c +++ b/lib/xz/xz_dec_bcj.c @@ -23,7 +23,8 @@ struct xz_dec_bcj { BCJ_IA64 = 6, /* Big or little endian */ BCJ_ARM = 7, /* Little endian only */ BCJ_ARMTHUMB = 8, /* Little endian only */ - BCJ_SPARC = 9 /* Big or little endian */ + BCJ_SPARC = 9, /* Big or little endian */ + BCJ_ARM64 = 10 /* AArch64 */ } type; /* @@ -346,6 +347,47 @@ static size_t bcj_sparc(struct xz_dec_bcj *s, uint8_t *buf, size_t size) } #endif +#ifdef XZ_DEC_ARM64 +static size_t bcj_arm64(struct xz_dec_bcj *s, uint8_t *buf, size_t size) +{ + size_t i; + uint32_t instr; + uint32_t addr; + + size &= ~(size_t)3; + + for (i = 0; i < size; i += 4) { + instr = get_unaligned_le32(buf + i); + + if ((instr >> 26) == 0x25) { + /* BL instruction */ + addr = instr - ((s->pos + (uint32_t)i) >> 2); + instr = 0x94000000 | (addr & 0x03FFFFFF); + put_unaligned_le32(instr, buf + i); + + } else if ((instr & 0x9F000000) == 0x90000000) { + /* ADRP instruction */ + addr = ((instr >> 29) & 3) | ((instr >> 3) & 0x1FFFFC); + + /* Only convert values in the range +/-512 MiB. */ + if ((addr + 0x020000) & 0x1C0000) + continue; + + addr -= (s->pos + (uint32_t)i) >> 12; + + instr &= 0x9000001F; + instr |= (addr & 3) << 29; + instr |= (addr & 0x03FFFC) << 3; + instr |= (0U - (addr & 0x020000)) & 0xE00000; + + put_unaligned_le32(instr, buf + i); + } + } + + return i; +} +#endif + /* * Apply the selected BCJ filter. Update *pos and s->pos to match the amount * of data that got filtered. @@ -392,6 +434,11 @@ static void bcj_apply(struct xz_dec_bcj *s, case BCJ_SPARC: filtered = bcj_sparc(s, buf, size); break; +#endif +#ifdef XZ_DEC_ARM64 + case BCJ_ARM64: + filtered = bcj_arm64(s, buf, size); + break; #endif default: /* Never reached but silence compiler warnings. */ @@ -565,6 +612,9 @@ XZ_EXTERN enum xz_ret xz_dec_bcj_reset(struct xz_dec_bcj *s, uint8_t id) #endif #ifdef XZ_DEC_SPARC case BCJ_SPARC: +#endif +#ifdef XZ_DEC_ARM64 + case BCJ_ARM64: #endif break; diff --git a/lib/xz/xz_private.h b/lib/xz/xz_private.h index 811add814ae4..307e0de8c260 100644 --- a/lib/xz/xz_private.h +++ b/lib/xz/xz_private.h @@ -36,6 +36,9 @@ # ifdef CONFIG_XZ_DEC_SPARC # define XZ_DEC_SPARC # endif +# ifdef CONFIG_XZ_DEC_ARM64 +# define XZ_DEC_ARM64 +# endif # ifdef CONFIG_XZ_DEC_MICROLZMA # define XZ_DEC_MICROLZMA # endif @@ -97,9 +100,9 @@ */ #ifndef XZ_DEC_BCJ # if defined(XZ_DEC_X86) || defined(XZ_DEC_POWERPC) \ - || defined(XZ_DEC_IA64) || defined(XZ_DEC_ARM) \ + || defined(XZ_DEC_IA64) \ || defined(XZ_DEC_ARM) || defined(XZ_DEC_ARMTHUMB) \ - || defined(XZ_DEC_SPARC) + || defined(XZ_DEC_SPARC) || defined(XZ_DEC_ARM64) # define XZ_DEC_BCJ # endif #endif From 74254f296fea1bab347ab5a696f4761bfa22389b Mon Sep 17 00:00:00 2001 From: Lasse Collin Date: Sun, 21 Jul 2024 16:36:27 +0300 Subject: [PATCH 10/14] xz: add RISC-V BCJ filter mainline inclusion from mainline-v6.12-rc1 category: feature A later commit updates lib/decompress_unxz.c to enable this filter for kernel decompression. lib/decompress_unxz.c is already used if CONFIG_EFI_ZBOOT=y && CONFIG_KERNEL_XZ=y. This filter can be used by Squashfs without modifications to the Squashfs kernel code (only needs support in userspace Squashfs-tools). Link: https://lkml.kernel.org/r/20240721133633.47721-13-lasse.collin@tukaani.org Signed-off-by: Lasse Collin Reviewed-by: Sam James Cc: Albert Ou Cc: Catalin Marinas Cc: Emil Renner Berthing Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Joel Stanley Cc: Jonathan Corbet Cc: Jubin Zhong Cc: Jules Maselbas Cc: Krzysztof Kozlowski Cc: Michael Ellerman Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Randy Dunlap Cc: Rui Li Cc: Simon Glass Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton (cherry picked from commit 93d09773d1a5339160e23906c68c42644e13e3d8) --- lib/xz/Kconfig | 5 +++ lib/xz/xz_dec_bcj.c | 104 ++++++++++++++++++++++++++++++++++++++++++- lib/xz/xz_dec_syms.c | 2 +- lib/xz/xz_private.h | 6 ++- 4 files changed, 114 insertions(+), 3 deletions(-) diff --git a/lib/xz/Kconfig b/lib/xz/Kconfig index ac3fd920da7d..8968e2b8a12e 100644 --- a/lib/xz/Kconfig +++ b/lib/xz/Kconfig @@ -45,6 +45,11 @@ config XZ_DEC_SPARC default y select XZ_DEC_BCJ +config XZ_DEC_RISCV + bool "RISC-V BCJ filter decoder" if EXPERT + default y + select XZ_DEC_BCJ + config XZ_DEC_MICROLZMA bool "MicroLZMA decoder" default n diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c index 941198a8a55b..42d7f268726f 100644 --- a/lib/xz/xz_dec_bcj.c +++ b/lib/xz/xz_dec_bcj.c @@ -24,7 +24,8 @@ struct xz_dec_bcj { BCJ_ARM = 7, /* Little endian only */ BCJ_ARMTHUMB = 8, /* Little endian only */ BCJ_SPARC = 9, /* Big or little endian */ - BCJ_ARM64 = 10 /* AArch64 */ + BCJ_ARM64 = 10, /* AArch64 */ + BCJ_RISCV = 11 /* RV32GQC_Zfh, RV64GQC_Zfh */ } type; /* @@ -388,6 +389,99 @@ static size_t bcj_arm64(struct xz_dec_bcj *s, uint8_t *buf, size_t size) } #endif +#ifdef XZ_DEC_RISCV +static size_t bcj_riscv(struct xz_dec_bcj *s, uint8_t *buf, size_t size) +{ + size_t i; + uint32_t b1; + uint32_t b2; + uint32_t b3; + uint32_t instr; + uint32_t instr2; + uint32_t instr2_rs1; + uint32_t addr; + + if (size < 8) + return 0; + + size -= 8; + + for (i = 0; i <= size; i += 2) { + instr = buf[i]; + + if (instr == 0xEF) { + /* JAL */ + b1 = buf[i + 1]; + if ((b1 & 0x0D) != 0) + continue; + + b2 = buf[i + 2]; + b3 = buf[i + 3]; + + addr = ((b1 & 0xF0) << 13) | (b2 << 9) | (b3 << 1); + addr -= s->pos + (uint32_t)i; + + buf[i + 1] = (uint8_t)((b1 & 0x0F) + | ((addr >> 8) & 0xF0)); + + buf[i + 2] = (uint8_t)(((addr >> 16) & 0x0F) + | ((addr >> 7) & 0x10) + | ((addr << 4) & 0xE0)); + + buf[i + 3] = (uint8_t)(((addr >> 4) & 0x7F) + | ((addr >> 13) & 0x80)); + + i += 4 - 2; + + } else if ((instr & 0x7F) == 0x17) { + /* AUIPC */ + instr |= (uint32_t)buf[i + 1] << 8; + instr |= (uint32_t)buf[i + 2] << 16; + instr |= (uint32_t)buf[i + 3] << 24; + + if (instr & 0xE80) { + /* AUIPC's rd doesn't equal x0 or x2. */ + instr2 = get_unaligned_le32(buf + i + 4); + + if (((instr << 8) ^ (instr2 - 3)) & 0xF8003) { + i += 6 - 2; + continue; + } + + addr = (instr & 0xFFFFF000) + (instr2 >> 20); + + instr = 0x17 | (2 << 7) | (instr2 << 12); + instr2 = addr; + } else { + /* AUIPC's rd equals x0 or x2. */ + instr2_rs1 = instr >> 27; + + if ((uint32_t)((instr - 0x3117) << 18) + >= (instr2_rs1 & 0x1D)) { + i += 4 - 2; + continue; + } + + addr = get_unaligned_be32(buf + i + 4); + addr -= s->pos + (uint32_t)i; + + instr2 = (instr >> 12) | (addr << 20); + + instr = 0x17 | (instr2_rs1 << 7) + | ((addr + 0x800) & 0xFFFFF000); + } + + put_unaligned_le32(instr, buf + i); + put_unaligned_le32(instr2, buf + i + 4); + + i += 8 - 2; + } + } + + return i; +} +#endif + /* * Apply the selected BCJ filter. Update *pos and s->pos to match the amount * of data that got filtered. @@ -439,6 +533,11 @@ static void bcj_apply(struct xz_dec_bcj *s, case BCJ_ARM64: filtered = bcj_arm64(s, buf, size); break; +#endif +#ifdef XZ_DEC_RISCV + case BCJ_RISCV: + filtered = bcj_riscv(s, buf, size); + break; #endif default: /* Never reached but silence compiler warnings. */ @@ -615,6 +714,9 @@ XZ_EXTERN enum xz_ret xz_dec_bcj_reset(struct xz_dec_bcj *s, uint8_t id) #endif #ifdef XZ_DEC_ARM64 case BCJ_ARM64: +#endif +#ifdef XZ_DEC_RISCV + case BCJ_RISCV: #endif break; diff --git a/lib/xz/xz_dec_syms.c b/lib/xz/xz_dec_syms.c index 495d2cc2e6e8..f40817d65897 100644 --- a/lib/xz/xz_dec_syms.c +++ b/lib/xz/xz_dec_syms.c @@ -22,6 +22,6 @@ EXPORT_SYMBOL(xz_dec_microlzma_end); #endif MODULE_DESCRIPTION("XZ decompressor"); -MODULE_VERSION("1.1"); +MODULE_VERSION("1.2"); MODULE_AUTHOR("Lasse Collin and Igor Pavlov"); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/lib/xz/xz_private.h b/lib/xz/xz_private.h index 307e0de8c260..a8b1cbe8d21d 100644 --- a/lib/xz/xz_private.h +++ b/lib/xz/xz_private.h @@ -39,6 +39,9 @@ # ifdef CONFIG_XZ_DEC_ARM64 # define XZ_DEC_ARM64 # endif +# ifdef CONFIG_XZ_DEC_RISCV +# define XZ_DEC_RISCV +# endif # ifdef CONFIG_XZ_DEC_MICROLZMA # define XZ_DEC_MICROLZMA # endif @@ -102,7 +105,8 @@ # if defined(XZ_DEC_X86) || defined(XZ_DEC_POWERPC) \ || defined(XZ_DEC_IA64) \ || defined(XZ_DEC_ARM) || defined(XZ_DEC_ARMTHUMB) \ - || defined(XZ_DEC_SPARC) || defined(XZ_DEC_ARM64) + || defined(XZ_DEC_SPARC) || defined(XZ_DEC_ARM64) \ + || defined(XZ_DEC_RISCV) # define XZ_DEC_BCJ # endif #endif From 3e710fa46ef6a31428db2c482178489f03a02636 Mon Sep 17 00:00:00 2001 From: Lasse Collin Date: Sun, 21 Jul 2024 16:36:28 +0300 Subject: [PATCH 11/14] xz: use 128 MiB dictionary and force single-threaded mode mainline inclusion from mainline-v6.12-rc1 category: feature This only affects kernel image compression, not any other xz usage. Desktop kernels on x86-64 are already around 60 MiB. Using a dictionary larger than 32 MiB should have no downsides nowadays as anyone building the kernel should have plenty of RAM. 128 MiB dictionary needs 1346 MiB of RAM with xz versions 5.0.x - 5.6.x in single-threaded mode. On archs that use xz_wrap.sh, kernel decompression is done in single-call mode so a larger dictionary doesn't affect boot-time memory requirements. xz >= 5.6.0 uses multithreaded mode by default which compresses slightly worse than single-threaded mode. Kernel compression rarely used more than one thread anyway because with 32 MiB dictionary size the default block size was 96 MiB in multithreaded mode. So only a single thread was used anyway unless the kernel was over 96 MiB. Comparison to CONFIG_KERNEL_LZMA: It uses "lzma -9" which mapped to 32 MiB dictionary in LZMA Utils 4.32.7 (the final release in 2008). Nowadays the lzma tool on most systems is from XZ Utils where -9 maps to 64 MiB dictionary. So using a 32 MiB dictionary with CONFIG_KERNEL_XZ may have compressed big kernels slightly worse than the old LZMA option. Comparison to CONFIG_KERNEL_ZSTD: zstd uses 128 MiB dictionary. Link: https://lkml.kernel.org/r/20240721133633.47721-14-lasse.collin@tukaani.org Signed-off-by: Lasse Collin Reviewed-by: Sam James Cc: Albert Ou Cc: Catalin Marinas Cc: Emil Renner Berthing Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Joel Stanley Cc: Jonathan Corbet Cc: Jubin Zhong Cc: Jules Maselbas Cc: Krzysztof Kozlowski Cc: Michael Ellerman Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Randy Dunlap Cc: Rui Li Cc: Simon Glass Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton (cherry picked from commit 8653c909922743bceb4800e5cc26087208c9e0e6) --- scripts/xz_wrap.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/xz_wrap.sh b/scripts/xz_wrap.sh index c06ff99b0af7..cc7377f23af3 100755 --- a/scripts/xz_wrap.sh +++ b/scripts/xz_wrap.sh @@ -17,4 +17,15 @@ case $SRCARCH in sparc) BCJ=--sparc ;; esac -exec $XZ --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB +# Use single-threaded mode because it compresses a little better +# (and uses less RAM) than multithreaded mode. +# +# For the best compression, the dictionary size shouldn't be +# smaller than the uncompressed kernel. 128 MiB dictionary +# needs less than 1400 MiB of RAM in single-threaded mode. +# +# On the archs that use this script to compress the kernel, +# decompression in the preboot code is done in single-call mode. +# Thus the dictionary size doesn't affect the memory requirements +# of the preboot decompressor at all. +exec $XZ --check=crc32 --threads=1 $BCJ --lzma2=$LZMA2OPTS,dict=128MiB From 71841c0e86b5933820369babd5ef5963dde3d3de Mon Sep 17 00:00:00 2001 From: Wentao Guan Date: Tue, 14 Jan 2025 17:48:34 +0800 Subject: [PATCH 12/14] xz: adjust arch-specific options for better kernel compression mainline inclusion from mainline-v6.12-rc1 category: feature Confilct: remove ia64) BCJ=--ia64; LZMA2OPTS=pb=4 ;; 0df8e970859: ("scripts: clean up IA-64 code") Use LZMA2 options that match the arch-specific alignment of instructions. This change reduces compressed kernel size 0-2 % depending on the arch. On 1-byte-aligned x86 it makes no difference and on 4-byte-aligned archs it helps the most. Use the ARM-Thumb filter for ARM-Thumb2 kernels. This reduces compressed kernel size about 5 %.[1] Previously such kernels were compressed using the ARM filter which didn't do anything useful with ARM-Thumb2 code. Add BCJ filter support for ARM64 and RISC-V. Compared to unfiltered XZ or plain LZMA, the compressed kernel size is reduced about 5 % on ARM64 and 7 % on RISC-V. A new enough version of the xz tool is required: 5.4.0 for ARM64 and 5.6.0 for RISC-V. With an old xz version, a message is printed to standard error and the kernel is compressed without the filter. Update lib/decompress_unxz.c to match the changes to xz_wrap.sh. Update the CONFIG_KERNEL_XZ help text in init/Kconfig: - Add the RISC-V and ARM64 filters. - Clarify that the PowerPC filter is for big endian only. - Omit IA-64. Link: https://lore.kernel.org/lkml/1637379771-39449-1-git-send-email-zhongjubin@huawei.com/ [1] Link: https://lkml.kernel.org/r/20240721133633.47721-15-lasse.collin@tukaani.org Signed-off-by: Lasse Collin Reviewed-by: Sam James Cc: Simon Glass Cc: Catalin Marinas Cc: Will Deacon Cc: Paul Walmsley Cc: Palmer Dabbelt Cc: Albert Ou Cc: Jubin Zhong Cc: Jules Maselbas Cc: Emil Renner Berthing Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Joel Stanley Cc: Jonathan Corbet Cc: Krzysztof Kozlowski Cc: Michael Ellerman Cc: Randy Dunlap Cc: Rui Li Cc: Thomas Gleixner Signed-off-by: Andrew Morton (cherry picked from commit 7472ff8adad8655f38b060a602f66e59c93c4793) Signed-off-by: Wentao Guan --- init/Kconfig | 5 +- lib/decompress_unxz.c | 14 ++++- scripts/xz_wrap.sh | 143 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 152 insertions(+), 10 deletions(-) diff --git a/init/Kconfig b/init/Kconfig index 50798796b616..0a81331b1e0d 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -317,8 +317,9 @@ config KERNEL_XZ BCJ filters which can improve compression ratio of executable code. The size of the kernel is about 30% smaller with XZ in comparison to gzip. On architectures for which there is a BCJ - filter (i386, x86_64, ARM, IA-64, PowerPC, and SPARC), XZ - will create a few percent smaller kernel than plain LZMA. + filter (i386, x86_64, ARM, ARM64, RISC-V, big endian PowerPC, + and SPARC), XZ will create a few percent smaller kernel than + plain LZMA. The speed is about the same as with LZMA: The decompression speed of XZ is better than that of bzip2 but worse than gzip diff --git a/lib/decompress_unxz.c b/lib/decompress_unxz.c index 86a07f2af601..014c35ff6799 100644 --- a/lib/decompress_unxz.c +++ b/lib/decompress_unxz.c @@ -126,11 +126,21 @@ #ifdef CONFIG_X86 # define XZ_DEC_X86 #endif -#ifdef CONFIG_PPC +#if defined(CONFIG_PPC) && defined(CONFIG_CPU_BIG_ENDIAN) # define XZ_DEC_POWERPC #endif #ifdef CONFIG_ARM -# define XZ_DEC_ARM +# ifdef CONFIG_THUMB2_KERNEL +# define XZ_DEC_ARMTHUMB +# else +# define XZ_DEC_ARM +# endif +#endif +#ifdef CONFIG_ARM64 +# define XZ_DEC_ARM64 +#endif +#ifdef CONFIG_RISCV +# define XZ_DEC_RISCV #endif #ifdef CONFIG_IA64 # define XZ_DEC_IA64 diff --git a/scripts/xz_wrap.sh b/scripts/xz_wrap.sh index cc7377f23af3..f19369687030 100755 --- a/scripts/xz_wrap.sh +++ b/scripts/xz_wrap.sh @@ -6,15 +6,146 @@ # # Author: Lasse Collin +# This has specialized settings for the following archs. However, +# XZ-compressed kernel isn't currently supported on every listed arch. +# +# Arch Align Notes +# arm 2/4 ARM and ARM-Thumb2 +# arm64 4 +# csky 2 +# loongarch 4 +# mips 2/4 MicroMIPS is 2-byte aligned +# parisc 4 +# powerpc 4 Uses its own wrapper for compressors instead of this. +# riscv 2/4 +# s390 2 +# sh 2 +# sparc 4 +# x86 1 + +# A few archs use 2-byte or 4-byte aligned instructions depending on +# the kernel config. This function is used to check if the relevant +# config option is set to "y". +is_enabled() +{ + grep -q "^$1=y$" include/config/auto.conf +} + +# XZ_VERSION is needed to disable features that aren't available in +# old XZ Utils versions. +XZ_VERSION=$($XZ --robot --version) || exit +XZ_VERSION=$(printf '%s\n' "$XZ_VERSION" | sed -n 's/^XZ_VERSION=//p') + +# Assume that no BCJ filter is available. BCJ= -LZMA2OPTS= +# Set the instruction alignment to 1, 2, or 4 bytes. +# +# Set the BCJ filter if one is available. +# It must match the #ifdef usage in lib/decompress_unxz.c. case $SRCARCH in - x86) BCJ=--x86 ;; - powerpc) BCJ=--powerpc ;; - ia64) BCJ=--ia64; LZMA2OPTS=pb=4 ;; - arm) BCJ=--arm ;; - sparc) BCJ=--sparc ;; + arm) + if is_enabled CONFIG_THUMB2_KERNEL; then + ALIGN=2 + BCJ=--armthumb + else + ALIGN=4 + BCJ=--arm + fi + ;; + + arm64) + ALIGN=4 + + # ARM64 filter was added in XZ Utils 5.4.0. + if [ "$XZ_VERSION" -ge 50040002 ]; then + BCJ=--arm64 + else + echo "$0: Upgrading to xz >= 5.4.0" \ + "would enable the ARM64 filter" \ + "for better compression" >&2 + fi + ;; + + csky) + ALIGN=2 + ;; + + loongarch) + ALIGN=4 + ;; + + mips) + if is_enabled CONFIG_CPU_MICROMIPS; then + ALIGN=2 + else + ALIGN=4 + fi + ;; + + parisc) + ALIGN=4 + ;; + + powerpc) + ALIGN=4 + + # The filter is only for big endian instruction encoding. + if is_enabled CONFIG_CPU_BIG_ENDIAN; then + BCJ=--powerpc + fi + ;; + + riscv) + if is_enabled CONFIG_RISCV_ISA_C; then + ALIGN=2 + else + ALIGN=4 + fi + + # RISC-V filter was added in XZ Utils 5.6.0. + if [ "$XZ_VERSION" -ge 50060002 ]; then + BCJ=--riscv + else + echo "$0: Upgrading to xz >= 5.6.0" \ + "would enable the RISC-V filter" \ + "for better compression" >&2 + fi + ;; + + s390) + ALIGN=2 + ;; + + sh) + ALIGN=2 + ;; + + sparc) + ALIGN=4 + BCJ=--sparc + ;; + + x86) + ALIGN=1 + BCJ=--x86 + ;; + + *) + echo "$0: Arch-specific tuning is missing for '$SRCARCH'" >&2 + + # Guess 2-byte-aligned instructions. Guessing too low + # should hurt less than guessing too high. + ALIGN=2 + ;; +esac + +# Select the LZMA2 options matching the instruction alignment. +case $ALIGN in + 1) LZMA2OPTS= ;; + 2) LZMA2OPTS=lp=1 ;; + 4) LZMA2OPTS=lp=2,lc=2 ;; + *) echo "$0: ALIGN wrong or missing" >&2; exit 1 ;; esac # Use single-threaded mode because it compresses a little better From 980313b33861f9c3218ebf991f5d73885c66e1b9 Mon Sep 17 00:00:00 2001 From: Lasse Collin Date: Sun, 21 Jul 2024 16:36:30 +0300 Subject: [PATCH 13/14] arm64: boot: add Image.xz support mainline inclusion from mainline-v6.12-rc1 category: feature Confilct: rework for FIT image support https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=7a23b027ec17b2eb9c8ad9b09006502f3fa38215 The Image.* targets existed for other compressors already. Bootloader support is needed for decompression. This is for CONFIG_EFI_ZBOOT=n. With CONFIG_EFI_ZBOOT=y, XZ was already available. Link: https://lkml.kernel.org/r/20240721133633.47721-16-lasse.collin@tukaani.org Signed-off-by: Lasse Collin Cc: Simon Glass Cc: Catalin Marinas Cc: Will Deacon Cc: Jules Maselbas Cc: Albert Ou Cc: Emil Renner Berthing Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Joel Stanley Cc: Jonathan Corbet Cc: Jubin Zhong Cc: Krzysztof Kozlowski Cc: Michael Ellerman Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Randy Dunlap Cc: Rui Li Cc: Sam James Cc: Thomas Gleixner Signed-off-by: Andrew Morton (cherry picked from commit 181e71f6626ce04122ed04fa5f4de6726c1ac848) --- arch/arm64/boot/Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/Makefile b/arch/arm64/boot/Makefile index 1761f5972443..bf532dcc5ba3 100644 --- a/arch/arm64/boot/Makefile +++ b/arch/arm64/boot/Makefile @@ -16,7 +16,8 @@ OBJCOPYFLAGS_Image :=-O binary -R .note -R .note.gnu.build-id -R .comment -S -targets := Image Image.bz2 Image.gz Image.lz4 Image.lzma Image.lzo Image.zst +targets := Image Image.bz2 Image.gz Image.lz4 Image.lzma Image.lzo \ + Image.zst Image.xz $(obj)/Image: vmlinux FORCE $(call if_changed,objcopy) @@ -39,6 +40,9 @@ $(obj)/Image.lzo: $(obj)/Image FORCE $(obj)/Image.zst: $(obj)/Image FORCE $(call if_changed,zstd) +$(obj)/Image.xz: $(obj)/Image FORCE + $(call if_changed,xzkern) + EFI_ZBOOT_PAYLOAD := Image EFI_ZBOOT_BFD_TARGET := elf64-littleaarch64 EFI_ZBOOT_MACH_TYPE := ARM64 From 2dfe3ea689c4b6b788d3711ed9d3a46334525abe Mon Sep 17 00:00:00 2001 From: Lasse Collin Date: Wed, 24 Jul 2024 14:05:41 +0300 Subject: [PATCH 14/14] xz: remove XZ_EXTERN and extern from functions mainline inclusion from mainline-v6.12-rc1 category: feature XZ_EXTERN was used to make internal functions static in the preboot code. However, in other decompressors this hasn't been done. On x86-64, this makes no difference to the kernel image size. Omit XZ_EXTERN and let some of the internal functions be extern in the preboot code. Omitting XZ_EXTERN from include/linux/xz.h fixes warnings in "make htmldocs" and makes the intradocument links to xz_dec functions work in Documentation/staging/xz.rst. The alternative would have been to add "XZ_EXTERN" to c_id_attributes in Documentation/conf.py but omitting XZ_EXTERN seemed cleaner. Link: https://lore.kernel.org/lkml/20240723205437.3c0664b0@kaneli/ Link: https://lkml.kernel.org/r/20240724110544.16430-1-lasse.collin@tukaani.org Signed-off-by: Lasse Collin Tested-by: Michael Ellerman (powerpc) Cc: Jonathan Corbet Cc: Sam James Cc: Albert Ou Cc: Catalin Marinas Cc: Emil Renner Berthing Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Joel Stanley Cc: Jubin Zhong Cc: Jules Maselbas Cc: Krzysztof Kozlowski Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Randy Dunlap Cc: Rui Li Cc: Simon Glass Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton (cherry picked from commit c6f371bab25edccd39caa5dd452b50d9dfdf4ff0) Signed-off-by: Wentao Guan --- Documentation/staging/xz.rst | 3 --- arch/powerpc/boot/xz_config.h | 3 --- include/linux/xz.h | 35 ++++++++++++----------------------- lib/decompress_unxz.c | 1 - lib/xz/xz_crc32.c | 4 ++-- lib/xz/xz_dec_bcj.c | 9 ++++----- lib/xz/xz_dec_lzma2.c | 10 ++++------ lib/xz/xz_dec_stream.c | 8 ++++---- lib/xz/xz_private.h | 20 ++++++++------------ 9 files changed, 34 insertions(+), 59 deletions(-) diff --git a/Documentation/staging/xz.rst b/Documentation/staging/xz.rst index e1054e9a8e65..6953a189e5f2 100644 --- a/Documentation/staging/xz.rst +++ b/Documentation/staging/xz.rst @@ -95,7 +95,4 @@ xz_dec API This is available with ``#include ``. -``XZ_EXTERN`` is a macro used in the preboot code. Ignore it when -reading this documentation. - .. kernel-doc:: include/linux/xz.h diff --git a/arch/powerpc/boot/xz_config.h b/arch/powerpc/boot/xz_config.h index ebfadd39e192..9506a96ebbcc 100644 --- a/arch/powerpc/boot/xz_config.h +++ b/arch/powerpc/boot/xz_config.h @@ -50,11 +50,8 @@ static inline void put_unaligned_be32(u32 val, void *p) /* prevent the inclusion of the xz-preboot MM headers */ #define DECOMPR_MM_H #define memmove memmove -#define XZ_EXTERN static /* xz.h needs to be included directly since we need enum xz_mode */ #include "../../../include/linux/xz.h" -#undef XZ_EXTERN - #endif diff --git a/include/linux/xz.h b/include/linux/xz.h index 701d62c02b9a..58ae1d746c6f 100644 --- a/include/linux/xz.h +++ b/include/linux/xz.h @@ -18,11 +18,6 @@ # include #endif -/* In Linux, this is used to make extern functions static when needed. */ -#ifndef XZ_EXTERN -# define XZ_EXTERN extern -#endif - /** * enum xz_mode - Operation mode * @@ -190,7 +185,7 @@ struct xz_dec; * ready to be used with xz_dec_run(). If memory allocation fails, * xz_dec_init() returns NULL. */ -XZ_EXTERN struct xz_dec *xz_dec_init(enum xz_mode mode, uint32_t dict_max); +struct xz_dec *xz_dec_init(enum xz_mode mode, uint32_t dict_max); /** * xz_dec_run() - Run the XZ decoder @@ -210,7 +205,7 @@ XZ_EXTERN struct xz_dec *xz_dec_init(enum xz_mode mode, uint32_t dict_max); * get that amount valid data from the beginning of the stream. You must use * the multi-call decoder if you don't want to uncompress the whole stream. */ -XZ_EXTERN enum xz_ret xz_dec_run(struct xz_dec *s, struct xz_buf *b); +enum xz_ret xz_dec_run(struct xz_dec *s, struct xz_buf *b); /** * xz_dec_reset() - Reset an already allocated decoder state @@ -223,14 +218,14 @@ XZ_EXTERN enum xz_ret xz_dec_run(struct xz_dec *s, struct xz_buf *b); * xz_dec_run(). Thus, explicit call to xz_dec_reset() is useful only in * multi-call mode. */ -XZ_EXTERN void xz_dec_reset(struct xz_dec *s); +void xz_dec_reset(struct xz_dec *s); /** * xz_dec_end() - Free the memory allocated for the decoder state * @s: Decoder state allocated using xz_dec_init(). If s is NULL, * this function does nothing. */ -XZ_EXTERN void xz_dec_end(struct xz_dec *s); +void xz_dec_end(struct xz_dec *s); /** * DOC: MicroLZMA decompressor @@ -244,10 +239,6 @@ XZ_EXTERN void xz_dec_end(struct xz_dec *s); * 3/0/2, the first byte is 0xA2. This way the first byte can never be 0x00. * Just like with LZMA2, lc + lp <= 4 must be true. The LZMA end-of-stream * marker must not be used. The unused values are reserved for future use. - * - * These functions aren't used or available in preboot code and thus aren't - * marked with XZ_EXTERN. This avoids warnings about static functions that - * are never defined. */ /* @@ -272,8 +263,8 @@ struct xz_dec_microlzma; * struct xz_dec_microlzma. If memory allocation fails or * dict_size is invalid, NULL is returned. */ -extern struct xz_dec_microlzma *xz_dec_microlzma_alloc(enum xz_mode mode, - uint32_t dict_size); +struct xz_dec_microlzma *xz_dec_microlzma_alloc(enum xz_mode mode, + uint32_t dict_size); /** * xz_dec_microlzma_reset() - Reset the MicroLZMA decoder state @@ -289,9 +280,8 @@ extern struct xz_dec_microlzma *xz_dec_microlzma_alloc(enum xz_mode mode, * requiring stdbool.h. This should normally be set to true. * When this is set to false, error detection is weaker. */ -extern void xz_dec_microlzma_reset(struct xz_dec_microlzma *s, - uint32_t comp_size, uint32_t uncomp_size, - int uncomp_size_is_exact); +void xz_dec_microlzma_reset(struct xz_dec_microlzma *s, uint32_t comp_size, + uint32_t uncomp_size, int uncomp_size_is_exact); /** * xz_dec_microlzma_run() - Run the MicroLZMA decoder @@ -329,15 +319,14 @@ extern void xz_dec_microlzma_reset(struct xz_dec_microlzma *s, * may be changed normally like with XZ_PREALLOC. This way input data can be * provided from non-contiguous memory. */ -extern enum xz_ret xz_dec_microlzma_run(struct xz_dec_microlzma *s, - struct xz_buf *b); +enum xz_ret xz_dec_microlzma_run(struct xz_dec_microlzma *s, struct xz_buf *b); /** * xz_dec_microlzma_end() - Free the memory allocated for the decoder state * @s: Decoder state allocated using xz_dec_microlzma_alloc(). * If s is NULL, this function does nothing. */ -extern void xz_dec_microlzma_end(struct xz_dec_microlzma *s); +void xz_dec_microlzma_end(struct xz_dec_microlzma *s); /* * Standalone build (userspace build or in-kernel build for boot time use) @@ -358,13 +347,13 @@ extern void xz_dec_microlzma_end(struct xz_dec_microlzma *s); * This must be called before any other xz_* function to initialize * the CRC32 lookup table. */ -XZ_EXTERN void xz_crc32_init(void); +void xz_crc32_init(void); /* * Update CRC32 value using the polynomial from IEEE-802.3. To start a new * calculation, the third argument must be zero. To continue the calculation, * the previously returned value is passed as the third argument. */ -XZ_EXTERN uint32_t xz_crc32(const uint8_t *buf, size_t size, uint32_t crc); +uint32_t xz_crc32(const uint8_t *buf, size_t size, uint32_t crc); #endif #endif diff --git a/lib/decompress_unxz.c b/lib/decompress_unxz.c index 014c35ff6799..4f4f82f7f434 100644 --- a/lib/decompress_unxz.c +++ b/lib/decompress_unxz.c @@ -107,7 +107,6 @@ #ifdef __KERNEL__ # include #endif -#define XZ_EXTERN STATIC #ifndef XZ_PREBOOT # include diff --git a/lib/xz/xz_crc32.c b/lib/xz/xz_crc32.c index effdf34ec48d..6a7906a328ba 100644 --- a/lib/xz/xz_crc32.c +++ b/lib/xz/xz_crc32.c @@ -26,7 +26,7 @@ STATIC_RW_DATA uint32_t xz_crc32_table[256]; -XZ_EXTERN void xz_crc32_init(void) +void xz_crc32_init(void) { const uint32_t poly = 0xEDB88320; @@ -45,7 +45,7 @@ XZ_EXTERN void xz_crc32_init(void) return; } -XZ_EXTERN uint32_t xz_crc32(const uint8_t *buf, size_t size, uint32_t crc) +uint32_t xz_crc32(const uint8_t *buf, size_t size, uint32_t crc) { crc = ~crc; diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c index 42d7f268726f..8237db17eee3 100644 --- a/lib/xz/xz_dec_bcj.c +++ b/lib/xz/xz_dec_bcj.c @@ -572,9 +572,8 @@ static void bcj_flush(struct xz_dec_bcj *s, struct xz_buf *b) * data in chunks of 1-16 bytes. To hide this issue, this function does * some buffering. */ -XZ_EXTERN enum xz_ret xz_dec_bcj_run(struct xz_dec_bcj *s, - struct xz_dec_lzma2 *lzma2, - struct xz_buf *b) +enum xz_ret xz_dec_bcj_run(struct xz_dec_bcj *s, struct xz_dec_lzma2 *lzma2, + struct xz_buf *b) { size_t out_start; @@ -682,7 +681,7 @@ XZ_EXTERN enum xz_ret xz_dec_bcj_run(struct xz_dec_bcj *s, return s->ret; } -XZ_EXTERN struct xz_dec_bcj *xz_dec_bcj_create(bool single_call) +struct xz_dec_bcj *xz_dec_bcj_create(bool single_call) { struct xz_dec_bcj *s = kmalloc(sizeof(*s), GFP_KERNEL); if (s != NULL) @@ -691,7 +690,7 @@ XZ_EXTERN struct xz_dec_bcj *xz_dec_bcj_create(bool single_call) return s; } -XZ_EXTERN enum xz_ret xz_dec_bcj_reset(struct xz_dec_bcj *s, uint8_t id) +enum xz_ret xz_dec_bcj_reset(struct xz_dec_bcj *s, uint8_t id) { switch (id) { #ifdef XZ_DEC_X86 diff --git a/lib/xz/xz_dec_lzma2.c b/lib/xz/xz_dec_lzma2.c index 613939f5dd6c..83bb66b6016d 100644 --- a/lib/xz/xz_dec_lzma2.c +++ b/lib/xz/xz_dec_lzma2.c @@ -960,8 +960,7 @@ static bool lzma2_lzma(struct xz_dec_lzma2 *s, struct xz_buf *b) * Take care of the LZMA2 control layer, and forward the job of actual LZMA * decoding or copying of uncompressed chunks to other functions. */ -XZ_EXTERN enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, - struct xz_buf *b) +enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b) { uint32_t tmp; @@ -1137,8 +1136,7 @@ XZ_EXTERN enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, return XZ_OK; } -XZ_EXTERN struct xz_dec_lzma2 *xz_dec_lzma2_create(enum xz_mode mode, - uint32_t dict_max) +struct xz_dec_lzma2 *xz_dec_lzma2_create(enum xz_mode mode, uint32_t dict_max) { struct xz_dec_lzma2 *s = kmalloc(sizeof(*s), GFP_KERNEL); if (s == NULL) @@ -1161,7 +1159,7 @@ XZ_EXTERN struct xz_dec_lzma2 *xz_dec_lzma2_create(enum xz_mode mode, return s; } -XZ_EXTERN enum xz_ret xz_dec_lzma2_reset(struct xz_dec_lzma2 *s, uint8_t props) +enum xz_ret xz_dec_lzma2_reset(struct xz_dec_lzma2 *s, uint8_t props) { /* This limits dictionary size to 3 GiB to keep parsing simpler. */ if (props > 39) @@ -1197,7 +1195,7 @@ XZ_EXTERN enum xz_ret xz_dec_lzma2_reset(struct xz_dec_lzma2 *s, uint8_t props) return XZ_OK; } -XZ_EXTERN void xz_dec_lzma2_end(struct xz_dec_lzma2 *s) +void xz_dec_lzma2_end(struct xz_dec_lzma2 *s) { if (DEC_IS_MULTI(s->dict.mode)) vfree(s->dict.buf); diff --git a/lib/xz/xz_dec_stream.c b/lib/xz/xz_dec_stream.c index 0058406ccd17..f9d003684d56 100644 --- a/lib/xz/xz_dec_stream.c +++ b/lib/xz/xz_dec_stream.c @@ -746,7 +746,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b) * actually succeeds (that's the price to pay of using the output buffer as * the workspace). */ -XZ_EXTERN enum xz_ret xz_dec_run(struct xz_dec *s, struct xz_buf *b) +enum xz_ret xz_dec_run(struct xz_dec *s, struct xz_buf *b) { size_t in_start; size_t out_start; @@ -782,7 +782,7 @@ XZ_EXTERN enum xz_ret xz_dec_run(struct xz_dec *s, struct xz_buf *b) return ret; } -XZ_EXTERN struct xz_dec *xz_dec_init(enum xz_mode mode, uint32_t dict_max) +struct xz_dec *xz_dec_init(enum xz_mode mode, uint32_t dict_max) { struct xz_dec *s = kmalloc(sizeof(*s), GFP_KERNEL); if (s == NULL) @@ -812,7 +812,7 @@ XZ_EXTERN struct xz_dec *xz_dec_init(enum xz_mode mode, uint32_t dict_max) return NULL; } -XZ_EXTERN void xz_dec_reset(struct xz_dec *s) +void xz_dec_reset(struct xz_dec *s) { s->sequence = SEQ_STREAM_HEADER; s->allow_buf_error = false; @@ -824,7 +824,7 @@ XZ_EXTERN void xz_dec_reset(struct xz_dec *s) s->temp.size = STREAM_HEADER_SIZE; } -XZ_EXTERN void xz_dec_end(struct xz_dec *s) +void xz_dec_end(struct xz_dec *s) { if (s != NULL) { xz_dec_lzma2_end(s->lzma2); diff --git a/lib/xz/xz_private.h b/lib/xz/xz_private.h index a8b1cbe8d21d..5f1294a1408c 100644 --- a/lib/xz/xz_private.h +++ b/lib/xz/xz_private.h @@ -115,8 +115,7 @@ * Allocate memory for LZMA2 decoder. xz_dec_lzma2_reset() must be used * before calling xz_dec_lzma2_run(). */ -XZ_EXTERN struct xz_dec_lzma2 *xz_dec_lzma2_create(enum xz_mode mode, - uint32_t dict_max); +struct xz_dec_lzma2 *xz_dec_lzma2_create(enum xz_mode mode, uint32_t dict_max); /* * Decode the LZMA2 properties (one byte) and reset the decoder. Return @@ -124,22 +123,20 @@ XZ_EXTERN struct xz_dec_lzma2 *xz_dec_lzma2_create(enum xz_mode mode, * big enough, and XZ_OPTIONS_ERROR if props indicates something that this * decoder doesn't support. */ -XZ_EXTERN enum xz_ret xz_dec_lzma2_reset(struct xz_dec_lzma2 *s, - uint8_t props); +enum xz_ret xz_dec_lzma2_reset(struct xz_dec_lzma2 *s, uint8_t props); /* Decode raw LZMA2 stream from b->in to b->out. */ -XZ_EXTERN enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, - struct xz_buf *b); +enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b); /* Free the memory allocated for the LZMA2 decoder. */ -XZ_EXTERN void xz_dec_lzma2_end(struct xz_dec_lzma2 *s); +void xz_dec_lzma2_end(struct xz_dec_lzma2 *s); #ifdef XZ_DEC_BCJ /* * Allocate memory for BCJ decoders. xz_dec_bcj_reset() must be used before * calling xz_dec_bcj_run(). */ -XZ_EXTERN struct xz_dec_bcj *xz_dec_bcj_create(bool single_call); +struct xz_dec_bcj *xz_dec_bcj_create(bool single_call); /* * Decode the Filter ID of a BCJ filter. This implementation doesn't @@ -147,16 +144,15 @@ XZ_EXTERN struct xz_dec_bcj *xz_dec_bcj_create(bool single_call); * is needed. Returns XZ_OK if the given Filter ID is supported. * Otherwise XZ_OPTIONS_ERROR is returned. */ -XZ_EXTERN enum xz_ret xz_dec_bcj_reset(struct xz_dec_bcj *s, uint8_t id); +enum xz_ret xz_dec_bcj_reset(struct xz_dec_bcj *s, uint8_t id); /* * Decode raw BCJ + LZMA2 stream. This must be used only if there actually is * a BCJ filter in the chain. If the chain has only LZMA2, xz_dec_lzma2_run() * must be called directly. */ -XZ_EXTERN enum xz_ret xz_dec_bcj_run(struct xz_dec_bcj *s, - struct xz_dec_lzma2 *lzma2, - struct xz_buf *b); +enum xz_ret xz_dec_bcj_run(struct xz_dec_bcj *s, struct xz_dec_lzma2 *lzma2, + struct xz_buf *b); /* Free the memory allocated for the BCJ filters. */ #define xz_dec_bcj_end(s) kfree(s)