[PATCH v1 4/4] compress: add lz4 decompression support

[He Yong]

Enable lz4 artifacts end-to-end by wiring parser mappings, runtime
decompression pipeline, build flags, and docs so compressed="lz4"
works like existing codecs.

Signed-off-by: He Yong <hyy...@163.com>
---
Kconfig | 8 +++
Makefile.deps | 4 ++
Makefile.flags | 4 ++
ci/setup.sh | 1 +
core/cpio_utils.c | 101 +++++++++++++++++++++++++++++++++-
core/util.c | 1 +
doc/source/sw-description.rst | 2 +-
doc/source/swupdate.rst | 2 +-
handlers/swupdate.lua | 2 +-
include/util.h | 1 +
10 files changed, 120 insertions(+), 6 deletions(-)

diff --git a/Kconfig b/Kconfig
index 2cf68eb8..8b6139de 100644
--- a/Kconfig
+++ b/Kconfig
@@ -85,6 +85,10 @@ config HAVE_ZSTD
bool
option env="HAVE_ZSTD"

+config HAVE_LZ4
+ bool
+ option env="HAVE_LZ4"
+
config HAVE_LIBSSL
bool
option env="HAVE_LIBSSL"
@@ -421,6 +425,10 @@ config ZSTD
bool "Zstd compression support"
depends on HAVE_ZSTD

+config LZ4
+ bool "LZ4 compression support"
+ depends on HAVE_LZ4
+
comment "Parsers"
source "parser/Kconfig"

diff --git a/Makefile.deps b/Makefile.deps
index c759f687..35f1bd6b 100644
--- a/Makefile.deps
+++ b/Makefile.deps
@@ -66,6 +66,10 @@ ifeq ($(HAVE_ZSTD),)
export HAVE_ZSTD = y
endif

+ifeq ($(HAVE_LZ4),)
+export HAVE_LZ4 = y
+endif
+
ifeq ($(HAVE_LIBEXT2FS),)
export HAVE_LIBEXT2FS = y
endif
diff --git a/Makefile.flags b/Makefile.flags
index 40dd3b66..267d74ab 100644
--- a/Makefile.flags
+++ b/Makefile.flags
@@ -197,6 +197,10 @@ ifeq ($(CONFIG_ZSTD),y)
LDLIBS += zstd
endif

+ifeq ($(CONFIG_LZ4),y)
+LDLIBS += lz4
+endif
+
ifeq ($(CONFIG_DISKPART),y)
LDLIBS += fdisk
endif
diff --git a/ci/setup.sh b/ci/setup.sh
index 863b1c42..82aca63a 100755
--- a/ci/setup.sh
+++ b/ci/setup.sh
@@ -51,6 +51,7 @@ $_SUDO apt-get -qq update && apt-get install --yes --no-install-recommends \
liburiparser-dev \
libwebsockets-dev \
liblzma-dev \
+ liblz4-dev \
libwolfssl-dev \
libzstd-dev \
make \
diff --git a/core/cpio_utils.c b/core/cpio_utils.c
index cb291d3b..4a8b2964 100644
--- a/core/cpio_utils.c
+++ b/core/cpio_utils.c
@@ -22,6 +22,9 @@
#ifdef CONFIG_ZSTD
#include <zstd.h>
#endif
+#ifdef CONFIG_LZ4
+#include <lz4frame.h>
+#endif

#include "generated/autoconf.h"
#include "cpiohdr.h"
@@ -341,7 +344,7 @@ static int decrypt_step(void *state, void *buffer, size_t size)
return 0;
}

-#if defined(CONFIG_GUNZIP) || defined(CONFIG_ZSTD) || defined(CONFIG_XZ)
+#if defined(CONFIG_GUNZIP) || defined(CONFIG_ZSTD) || defined(CONFIG_XZ) || defined(CONFIG_LZ4)
typedef int (*DecompressStep)(void *state, void *buffer, size_t size);

struct DecompressState {
@@ -494,6 +497,70 @@ static int zstd_step(void* state, void* buffer, size_t size)

#endif

+#ifdef CONFIG_LZ4
+
+struct Lz4State {
+ LZ4F_decompressionContext_t dctx;
+ const uint8_t *input_src;
+ size_t input_size;
+ size_t input_pos;
+};
+
+static int lz4_step(void *state, void *buffer, size_t size)
+{
+ struct DecompressState *ds = (struct DecompressState *)state;
+ struct Lz4State *s = (struct Lz4State *)ds->impl_state;
+ size_t decompress_ret = 0;
+ size_t produced = 0;
+ int ret;
+
+ do {
+ if (s->input_pos == s->input_size) {
+ ret = ds->upstream_step(ds->upstream_state, ds->input, sizeof ds->input);
+ if (ret < 0) {
+ return ret;
+ } else if (ret == 0) {
+ ds->eof = true;
+ break;
+ }
+ s->input_src = ds->input;
+ s->input_size = ret;
+ s->input_pos = 0;
+ }
+
+ size_t out_len = size - produced;
+ size_t in_len = s->input_size - s->input_pos;
+ size_t old_produced = produced;
+ size_t old_in_pos = s->input_pos;
+
+ decompress_ret = LZ4F_decompress(s->dctx,
+ (uint8_t *)buffer + produced, &out_len,
+ s->input_src + s->input_pos, &in_len,
+ NULL);
+ if (LZ4F_isError(decompress_ret)) {
+ ERROR("LZ4F_decompress failed: %s", LZ4F_getErrorName(decompress_ret));
+ return -1;
+ }
+
+ produced += out_len;
+ s->input_pos += in_len;
+
+ if (old_produced == produced && old_in_pos == s->input_pos) {
+ ERROR("LZ4F_decompress made no progress");
+ return -1;
+ }
+
+ if (decompress_ret == 0) {
+ ds->eof = true;
+ break;
+ }
+ } while (produced == 0 && !ds->eof);
+
+ return produced;
+}
+
+#endif
+
static int hash_compare(void *dgst, unsigned char *hash)
{
/*
@@ -554,7 +621,7 @@ int copyfile(struct swupdate_copy *args)
.outlen = 0, .eof = false
};

-#if defined(CONFIG_GUNZIP) || defined(CONFIG_ZSTD) || defined(CONFIG_XZ)
+#if defined(CONFIG_GUNZIP) || defined(CONFIG_ZSTD) || defined(CONFIG_XZ) || defined(CONFIG_LZ4)
struct DecompressState decompress_state = {
.upstream_step = NULL, .upstream_state = NULL,
.impl_state = NULL
@@ -583,6 +650,14 @@ int copyfile(struct swupdate_copy *args)
.input_view = { NULL, 0, 0 },
};
#endif
+#ifdef CONFIG_LZ4
+ struct Lz4State lz4_state = {
+ .dctx = NULL,
+ .input_src = NULL,
+ .input_size = 0,
+ .input_pos = 0,
+ };
+#endif
#endif

/*
@@ -690,6 +765,21 @@ int copyfile(struct swupdate_copy *args)
decompress_step = &zstd_step;
decompress_state.impl_state = &zstd_state;
} else
+#endif
+#ifdef CONFIG_LZ4
+ if (args->compressed == COMPRESSED_LZ4) {
+ size_t create_ret;
+
+ create_ret = LZ4F_createDecompressionContext(&lz4_state.dctx, LZ4F_VERSION);
+ if (LZ4F_isError(create_ret)) {
+ ERROR("LZ4F_createDecompressionContext failed: %s",
+ LZ4F_getErrorName(create_ret));
+ ret = -EFAULT;
+ goto copyfile_exit;
+ }
+ decompress_step = &lz4_step;
+ decompress_state.impl_state = &lz4_state;
+ } else
#endif
{
TRACE("Requested decompression method (%d) is not configured!", args->compressed);
@@ -724,7 +814,7 @@ int copyfile(struct swupdate_copy *args)
state = &decrypt_state;
}

-#if defined(CONFIG_GUNZIP) || defined(CONFIG_ZSTD) || defined(CONFIG_XZ)
+#if defined(CONFIG_GUNZIP) || defined(CONFIG_ZSTD) || defined(CONFIG_XZ) || defined(CONFIG_LZ4)
if (args->compressed) {
decompress_state.upstream_step = step;
decompress_state.upstream_state = state;
@@ -806,6 +896,11 @@ copyfile_exit:
ZSTD_freeDStream(zstd_state.dctx);
}
#endif
+#ifdef CONFIG_LZ4
+ if (lz4_state.dctx != NULL) {
+ LZ4F_freeDecompressionContext(lz4_state.dctx);
+ }
+#endif

return ret;
}
diff --git a/core/util.c b/core/util.c
index 89046f3c..efc41f84 100644
--- a/core/util.c
+++ b/core/util.c
@@ -83,6 +83,7 @@ static const struct {
{ "zlib", COMPRESSED_ZLIB },
{ "xz", COMPRESSED_XZ },
{ "zstd", COMPRESSED_ZSTD },
+ { "lz4", COMPRESSED_LZ4 },
};

int compressed_string_to_type(const char *s, int *out_type)
diff --git a/doc/source/sw-description.rst b/doc/source/sw-description.rst
index b09333fc..2c1e0017 100644
--- a/doc/source/sw-description.rst
+++ b/doc/source/sw-description.rst
@@ -1491,7 +1491,7 @@ There are 4 main sections inside sw-description:
| | | | before being installed. the value |
| | | | denotes the compression type. |
| | | | currently supported values are "xz", |
- | | | | "zlib" and "zstd". |
+ | | | | "zlib", "zstd" and "lz4". |
+-------------+----------+------------+---------------------------------------+
| compressed | bool (dep| images | Deprecated. Use the string form. true |
| | recated) | files | is equal to 'compressed = "zlib"'. |
diff --git a/doc/source/swupdate.rst b/doc/source/swupdate.rst
index 84a8f21b..62681c2a 100644
--- a/doc/source/swupdate.rst
+++ b/doc/source/swupdate.rst
@@ -57,7 +57,7 @@ General Overview
SWUpdate can recreate UBI volumes, resizing them and
copying the new software.

-- support for compressed images, using the xz, zlib and zstd library.
+- support for compressed images, using the xz, zlib, zstd and lz4 library.
tarball (tgz file) are supported.

- support for partitioned USB-pen or unpartitioned (mainly
diff --git a/handlers/swupdate.lua b/handlers/swupdate.lua
index aaba4774..e4bda3fa 100644
--- a/handlers/swupdate.lua
+++ b/handlers/swupdate.lua
@@ -273,7 +273,7 @@ swupdate.register_handler = function(name, funcptr, mask) end
--- @field size number Artifact size
--- @field checksum number Computed checksum
--- @field skip number `skip_t` enum number as in `include/swupdate.h`
---- @field compressed string `zlib` or `zstd` (boolean value is deprecated)
+--- @field compressed string `zlib`, `xz`, `zstd` or `lz4` (boolean value is deprecated)
--- @field properties table Properties Table equivalent as specified in `sw-description`
--- @field sha256 string sha256 hash of the image, file, or script
local img_type = {
diff --git a/include/util.h b/include/util.h
index dcd0604c..5c7e1363 100644
--- a/include/util.h
+++ b/include/util.h
@@ -64,6 +64,7 @@ enum compression_type {
COMPRESSED_ZLIB,
COMPRESSED_XZ,
COMPRESSED_ZSTD,
+ COMPRESSED_LZ4,
};

typedef int (*writeimage) (void *out, const void *buf, size_t len);
--
2.43.0