From: Eric Biggers Date: Wed, 2 May 2018 03:40:55 +0000 (-0700) Subject: [EXPERIMENTAL, FOR BENCHMARKING ONLY] Zstandard compression support X-Git-Url: https://wimlib.net/git/?p=wimlib;a=commitdiff_plain;h=refs%2Fheads%2Fzstd_support [EXPERIMENTAL, FOR BENCHMARKING ONLY] Zstandard compression support --- diff --git a/Makefile.am b/Makefile.am index 603c8d9c..784dfc01 100644 --- a/Makefile.am +++ b/Makefile.am @@ -164,6 +164,11 @@ libwim_la_SOURCES += src/ntfs-3g_apply.c \ include/wimlib/ntfs_3g.h endif +if WITH_ZSTD +libwim_la_SOURCES += src/zstd_compress.c \ + src/zstd_decompress.c +endif + if WINDOWS_NATIVE_BUILD libwim_la_SOURCES += src/wimboot.c \ src/win32_common.c \ @@ -199,6 +204,7 @@ libwim_la_CFLAGS = \ $(LIBXML2_CFLAGS) \ $(LIBNTFS_3G_CFLAGS) \ $(LIBFUSE_CFLAGS) \ + $(LIBZSTD_CFLAGS) \ $(LIBCRYPTO_CFLAGS) libwim_la_LDFLAGS = $(AM_LDFLAGS) -version-info 29:0:14 @@ -209,6 +215,7 @@ libwim_la_LIBADD = \ $(LIBNTFS_3G_LIBS) \ $(LIBFUSE_LIBS) \ $(LIBRT_LIBS) \ + $(LIBZSTD_LIBS) \ $(LIBCRYPTO_LIBS) \ $(PLATFORM_LIBS) diff --git a/configure.ac b/configure.ac index 072da7d0..57dd78b8 100644 --- a/configure.ac +++ b/configure.ac @@ -221,6 +221,27 @@ else fi AM_CONDITIONAL([ENABLE_SSSE3_SHA1], [test "$ENABLE_SSSE3_SHA1" = "yes"]) +# --------------------------- Zstandard support ------------------------------- + +AC_MSG_CHECKING([whether to include support for Zstandard compression]) +AC_ARG_WITH([zstd], + [AS_HELP_STRING([--with-zstd], + [Zstandard compression support via libzstd + (EXPERIMENTAL)])], + [WITH_ZSTD=$withval], + [WITH_ZSTD=no]) +AC_MSG_RESULT([$WITH_ZSTD]) + +if test "$WITH_ZSTD" = "yes" ; then + PKG_CHECK_MODULES([LIBZSTD], [libzstd], [], + [AC_MSG_ERROR([Cannot find libzstd! Either install libzstd, or + configure --without-zstd to disable Zstandard compression + support.])]) + PKGCONFIG_PRIVATE_REQUIRES="$PKGCONFIG_PRIVATE_REQUIRES zstd" + AC_DEFINE([WITH_ZSTD], [1], [Define to 1 if using Zstandard support]) +fi +AM_CONDITIONAL([WITH_ZSTD], [test "$WITH_ZSTD" = "yes"]) + # ----------------------------- Other options --------------------------------- AC_MSG_CHECKING([whether to include error messages]) diff --git a/include/wimlib.h b/include/wimlib.h index 047716c0..8a1d71b8 100644 --- a/include/wimlib.h +++ b/include/wimlib.h @@ -568,6 +568,18 @@ enum wimlib_compression_type { * up to and including 2^30. */ WIMLIB_COMPRESSION_TYPE_LZMS = 3, + + /** + * The Zstandard compression format. This is a wimlib extension; + * Microsoft's WIMGAPI does not support this format. Zstandard supports + * chunk sizes from 4096 to 268435456 bytes, with the default being + * 131072. Multiple compression levels are supported as well. For + * consistency with wimlib's other compression types, the compression + * levels exposed by libzstd are multiplied by 5; e.g., wimlib's level + * 50 (the default level) corresponds to libzstd's level 10. libzstd + * allows levels 1 through 22. + */ + WIMLIB_COMPRESSION_TYPE_ZSTD = 4, }; /** @} */ diff --git a/include/wimlib/compressor_ops.h b/include/wimlib/compressor_ops.h index bef5cf6d..c8df2fb7 100644 --- a/include/wimlib/compressor_ops.h +++ b/include/wimlib/compressor_ops.h @@ -32,5 +32,6 @@ struct compressor_ops { extern const struct compressor_ops lzx_compressor_ops; extern const struct compressor_ops xpress_compressor_ops; extern const struct compressor_ops lzms_compressor_ops; +extern const struct compressor_ops zstd_compressor_ops; #endif /* _WIMLIB_COMPRESSOR_OPS_H */ diff --git a/include/wimlib/decompressor_ops.h b/include/wimlib/decompressor_ops.h index 44148b32..e51a671b 100644 --- a/include/wimlib/decompressor_ops.h +++ b/include/wimlib/decompressor_ops.h @@ -25,5 +25,6 @@ struct decompressor_ops { extern const struct decompressor_ops lzx_decompressor_ops; extern const struct decompressor_ops xpress_decompressor_ops; extern const struct decompressor_ops lzms_decompressor_ops; +extern const struct decompressor_ops zstd_decompressor_ops; #endif /* _WIMLIB_DECOMPRESSOR_OPS_H */ diff --git a/include/wimlib/header.h b/include/wimlib/header.h index 894bd225..32282835 100644 --- a/include/wimlib/header.h +++ b/include/wimlib/header.h @@ -191,4 +191,6 @@ struct wim_header { /* XPRESS, with small chunk size??? */ #define WIM_HDR_FLAG_COMPRESS_XPRESS_2 0x00200000 +#define WIM_HDR_FLAG_COMPRESS_ZSTD 0x00400000 + #endif /* _WIMLIB_HEADER_H */ diff --git a/programs/imagex.c b/programs/imagex.c index a10ac488..611934fa 100644 --- a/programs/imagex.c +++ b/programs/imagex.c @@ -524,6 +524,9 @@ print_available_compression_types(FILE *fp) " xpress (alias: \"fast\")\n" " lzx (alias: \"maximum\") (default for capture)\n" " lzms (alias: \"recovery\")\n" +#ifdef WITH_ZSTD + " zstd (alias: \"zstandard\")\n" +#endif "\n" ); tfputs(s, fp); @@ -572,6 +575,11 @@ T( ctype = WIMLIB_COMPRESSION_TYPE_LZMS; } else if (!tstrcasecmp(optarg, T("lzms"))) { ctype = WIMLIB_COMPRESSION_TYPE_LZMS; +#ifdef WITH_ZSTD + } else if (!tstrcasecmp(optarg, T("zstd")) || + !tstrcasecmp(optarg, T("zstandard"))) { + ctype = WIMLIB_COMPRESSION_TYPE_ZSTD; +#endif } else if (!tstrcasecmp(optarg, T("none"))) { ctype = WIMLIB_COMPRESSION_TYPE_NONE; } else { diff --git a/src/compress.c b/src/compress.c index 1b0d2c9f..3c2b3e8a 100644 --- a/src/compress.c +++ b/src/compress.c @@ -42,6 +42,9 @@ static const struct compressor_ops * const compressor_ops[] = { [WIMLIB_COMPRESSION_TYPE_XPRESS] = &xpress_compressor_ops, [WIMLIB_COMPRESSION_TYPE_LZX] = &lzx_compressor_ops, [WIMLIB_COMPRESSION_TYPE_LZMS] = &lzms_compressor_ops, +#ifdef WITH_ZSTD + [WIMLIB_COMPRESSION_TYPE_ZSTD] = &zstd_compressor_ops, +#endif }; /* Scale: 10 = low, 50 = medium, 100 = high */ diff --git a/src/decompress.c b/src/decompress.c index f255ea92..7d8747a8 100644 --- a/src/decompress.c +++ b/src/decompress.c @@ -40,6 +40,9 @@ static const struct decompressor_ops * const decompressor_ops[] = { [WIMLIB_COMPRESSION_TYPE_XPRESS] = &xpress_decompressor_ops, [WIMLIB_COMPRESSION_TYPE_LZX] = &lzx_decompressor_ops, [WIMLIB_COMPRESSION_TYPE_LZMS] = &lzms_decompressor_ops, +#ifdef WITH_ZSTD + [WIMLIB_COMPRESSION_TYPE_ZSTD] = &zstd_decompressor_ops, +#endif }; static bool diff --git a/src/header.c b/src/header.c index cf193963..8b17160b 100644 --- a/src/header.c +++ b/src/header.c @@ -213,6 +213,7 @@ static const struct { {WIM_HDR_FLAG_COMPRESS_XPRESS, "COMPRESS_XPRESS"}, {WIM_HDR_FLAG_COMPRESS_LZMS, "COMPRESS_LZMS"}, {WIM_HDR_FLAG_COMPRESS_XPRESS_2,"COMPRESS_XPRESS_2"}, + {WIM_HDR_FLAG_COMPRESS_ZSTD, "COMPRESS_ZSTD"}, }; /* API function documented in wimlib.h */ diff --git a/src/wim.c b/src/wim.c index ca40e53a..853a71e5 100644 --- a/src/wim.c +++ b/src/wim.c @@ -78,6 +78,15 @@ static const struct { .default_nonsolid_chunk_size = 131072, .default_solid_chunk_size = 67108864, }, +#ifdef WITH_ZSTD + [WIMLIB_COMPRESSION_TYPE_ZSTD] = { + .name = T("Zstandard"), + .min_chunk_size = 4096, + .max_chunk_size = 268435456, + .default_nonsolid_chunk_size = 131072, + .default_solid_chunk_size = 131072, + }, +#endif }; /* Is the specified compression type valid? */ @@ -707,6 +716,10 @@ begin_read(WIMStruct *wim, const void *wim_filename_or_fd, int open_flags) wim->compression_type = WIMLIB_COMPRESSION_TYPE_XPRESS; } else if (wim->hdr.flags & WIM_HDR_FLAG_COMPRESS_LZMS) { wim->compression_type = WIMLIB_COMPRESSION_TYPE_LZMS; +#ifdef WITH_ZSTD + } else if (wim->hdr.flags & WIM_HDR_FLAG_COMPRESS_ZSTD) { + wim->compression_type = WIMLIB_COMPRESSION_TYPE_ZSTD; +#endif } else { return WIMLIB_ERR_INVALID_COMPRESSION_TYPE; } diff --git a/src/write.c b/src/write.c index 4805913d..835ad41b 100644 --- a/src/write.c +++ b/src/write.c @@ -2767,6 +2767,10 @@ write_wim_part(WIMStruct *wim, case WIMLIB_COMPRESSION_TYPE_LZMS: wim->out_hdr.flags |= WIM_HDR_FLAG_COMPRESS_LZMS; break; + case WIMLIB_COMPRESSION_TYPE_ZSTD: + WARNING("Zstandard support is experimental -- use only for benchmarking!"); + wim->out_hdr.flags |= WIM_HDR_FLAG_COMPRESS_ZSTD; + break; } } diff --git a/src/zstd_compress.c b/src/zstd_compress.c new file mode 100644 index 00000000..a5ece8b1 --- /dev/null +++ b/src/zstd_compress.c @@ -0,0 +1,79 @@ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "wimlib/error.h" +#include "wimlib/compressor_ops.h" +#include "wimlib/util.h" + +#include + +struct zstd_compressor { + unsigned compression_level; + ZSTD_CCtx *cctx; +}; + +static u64 +zstd_get_needed_memory(size_t max_bufsize, unsigned compression_level, + bool destructive) +{ + return sizeof(struct zstd_compressor); +} + +static int +zstd_create_compressor(size_t max_bufsize, unsigned compression_level, + bool destructive, void **c_ret) +{ + struct zstd_compressor *c; + + c = MALLOC(sizeof(*c)); + if (!c) + goto oom; + + c->cctx = ZSTD_createCCtx(); + if (!c->cctx) + goto oom; + + c->compression_level = compression_level / 5; + c->compression_level = max(c->compression_level, 1); + c->compression_level = min(c->compression_level, ZSTD_maxCLevel()); + + *c_ret = c; + return 0; + +oom: + FREE(c); + return WIMLIB_ERR_NOMEM; +} + +static size_t +zstd_compress(const void *in, size_t in_nbytes, + void *out, size_t out_nbytes_avail, void *_c) +{ + struct zstd_compressor *c = _c; + size_t res; + + res = ZSTD_compressCCtx(c->cctx, out, out_nbytes_avail, in, in_nbytes, + c->compression_level); + + if (ZSTD_isError(res)) + return 0; + return res; +} + +static void +zstd_free_compressor(void *_c) +{ + struct zstd_compressor *c = _c; + + FREE(c->cctx); + FREE(c); +} + +const struct compressor_ops zstd_compressor_ops = { + .get_needed_memory = zstd_get_needed_memory, + .create_compressor = zstd_create_compressor, + .compress = zstd_compress, + .free_compressor = zstd_free_compressor, +}; + diff --git a/src/zstd_decompress.c b/src/zstd_decompress.c new file mode 100644 index 00000000..d2b14a9d --- /dev/null +++ b/src/zstd_decompress.c @@ -0,0 +1,40 @@ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "wimlib/error.h" +#include "wimlib/decompressor_ops.h" +#include "wimlib/util.h" + +#include + +static int +zstd_create_decompressor(size_t max_block_size, void **d_ret) +{ + *d_ret = ZSTD_createDCtx(); + if (!*d_ret) + return WIMLIB_ERR_NOMEM; + return 0; +} + +static int +zstd_decompress(const void *in, size_t in_nbytes, + void *out, size_t out_nbytes, void *_d) +{ + size_t res = ZSTD_decompressDCtx(_d, out, out_nbytes, in, in_nbytes); + if (res != out_nbytes) + return -1; + return 0; +} + +static void +zstd_free_decompressor(void *_d) +{ + ZSTD_freeDCtx(_d); +} + +const struct decompressor_ops zstd_decompressor_ops = { + .create_decompressor = zstd_create_decompressor, + .decompress = zstd_decompress, + .free_decompressor = zstd_free_decompressor, +};