[EXPERIMENTAL, FOR BENCHMARKING ONLY] Zstandard compression support

diff --git a/Makefile.am b/Makefile.am
index 603c8d9c1c8e03a897b01138b2bb467d9a45d9bb..784dfc016f0ddf7e5570c75f3b60df5fcd11140d 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -164,6 +164,11 @@ libwim_la_SOURCES += src/ntfs-3g_apply.c   \
                     include/wimlib/ntfs_3g.h
 endif
 
                     include/wimlib/ntfs_3g.h
 endif
 

+if WITH_ZSTD
+libwim_la_SOURCES += src/zstd_compress.c               \
+                    src/zstd_decompress.c
+endif
+

 if WINDOWS_NATIVE_BUILD
 libwim_la_SOURCES += src/wimboot.c                     \
                     src/win32_common.c                 \
 if WINDOWS_NATIVE_BUILD
 libwim_la_SOURCES += src/wimboot.c                     \
                     src/win32_common.c                 \


@@ -199,6 +204,7 @@ libwim_la_CFLAGS =          \
        $(LIBXML2_CFLAGS)       \
        $(LIBNTFS_3G_CFLAGS)    \
        $(LIBFUSE_CFLAGS)       \
        $(LIBXML2_CFLAGS)       \
        $(LIBNTFS_3G_CFLAGS)    \
        $(LIBFUSE_CFLAGS)       \

+       $(LIBZSTD_CFLAGS)       \

        $(LIBCRYPTO_CFLAGS)
 
 libwim_la_LDFLAGS = $(AM_LDFLAGS) -version-info 29:0:14
        $(LIBCRYPTO_CFLAGS)
 
 libwim_la_LDFLAGS = $(AM_LDFLAGS) -version-info 29:0:14


@@ -209,6 +215,7 @@ libwim_la_LIBADD =          \
        $(LIBNTFS_3G_LIBS)      \
        $(LIBFUSE_LIBS)         \
        $(LIBRT_LIBS)           \
        $(LIBNTFS_3G_LIBS)      \
        $(LIBFUSE_LIBS)         \
        $(LIBRT_LIBS)           \

+       $(LIBZSTD_LIBS)         \

        $(LIBCRYPTO_LIBS)       \
        $(PLATFORM_LIBS)
 
        $(LIBCRYPTO_LIBS)       \
        $(PLATFORM_LIBS)
 

diff --git a/configure.ac b/configure.ac
index 072da7d06407a1bc52269e7d16ac5181abd51b93..57dd78b80b4dce150d37b428ffab8b069cdc6bd7 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -221,6 +221,27 @@ else
 fi
 AM_CONDITIONAL([ENABLE_SSSE3_SHA1], [test "$ENABLE_SSSE3_SHA1" = "yes"])
 
 fi
 AM_CONDITIONAL([ENABLE_SSSE3_SHA1], [test "$ENABLE_SSSE3_SHA1" = "yes"])
 

+# --------------------------- Zstandard support -------------------------------
+
+AC_MSG_CHECKING([whether to include support for Zstandard compression])
+AC_ARG_WITH([zstd],
+           [AS_HELP_STRING([--with-zstd],
+                           [Zstandard compression support via libzstd
+                            (EXPERIMENTAL)])],
+           [WITH_ZSTD=$withval],
+           [WITH_ZSTD=no])
+AC_MSG_RESULT([$WITH_ZSTD])
+
+if test "$WITH_ZSTD" = "yes" ; then
+       PKG_CHECK_MODULES([LIBZSTD], [libzstd], [],
+               [AC_MSG_ERROR([Cannot find libzstd!  Either install libzstd, or
+                configure --without-zstd to disable Zstandard compression
+                support.])])
+       PKGCONFIG_PRIVATE_REQUIRES="$PKGCONFIG_PRIVATE_REQUIRES zstd"
+       AC_DEFINE([WITH_ZSTD], [1], [Define to 1 if using Zstandard support])
+fi
+AM_CONDITIONAL([WITH_ZSTD], [test "$WITH_ZSTD" = "yes"])
+

 # ----------------------------- Other options ---------------------------------
 
 AC_MSG_CHECKING([whether to include error messages])
 # ----------------------------- Other options ---------------------------------
 
 AC_MSG_CHECKING([whether to include error messages])

diff --git a/include/wimlib.h b/include/wimlib.h
index 047716c0338ea9887ca58f5b52127204b70457d1..8a1d71b8bb260cf16deaaf79837c77c6b29edbd8 100644 (file)
--- a/include/wimlib.h
+++ b/include/wimlib.h
@@ -568,6 +568,18 @@ enum wimlib_compression_type {
         * up to and including <c>2^30</c>.
         */
        WIMLIB_COMPRESSION_TYPE_LZMS = 3,
         * up to and including <c>2^30</c>.
         */
        WIMLIB_COMPRESSION_TYPE_LZMS = 3,

+
+       /**
+        * The Zstandard compression format.  This is a wimlib extension;
+        * Microsoft's WIMGAPI does not support this format.  Zstandard supports
+        * chunk sizes from 4096 to 268435456 bytes, with the default being
+        * 131072.  Multiple compression levels are supported as well.  For
+        * consistency with wimlib's other compression types, the compression
+        * levels exposed by libzstd are multiplied by 5; e.g., wimlib's level
+        * 50 (the default level) corresponds to libzstd's level 10.  libzstd
+        * allows levels 1 through 22.
+        */
+       WIMLIB_COMPRESSION_TYPE_ZSTD = 4,

 };
 
 /** @} */
 };
 
 /** @} */

diff --git a/include/wimlib/compressor_ops.h b/include/wimlib/compressor_ops.h
index bef5cf6d1b440d9f7d6e5b91adaf8052e530c4ee..c8df2fb7d683d6f80eeab19b103346c0271070d5 100644 (file)
--- a/include/wimlib/compressor_ops.h
+++ b/include/wimlib/compressor_ops.h
@@ -32,5 +32,6 @@ struct compressor_ops {
 extern const struct compressor_ops lzx_compressor_ops;
 extern const struct compressor_ops xpress_compressor_ops;
 extern const struct compressor_ops lzms_compressor_ops;
 extern const struct compressor_ops lzx_compressor_ops;
 extern const struct compressor_ops xpress_compressor_ops;
 extern const struct compressor_ops lzms_compressor_ops;

+extern const struct compressor_ops zstd_compressor_ops;

 
 #endif /* _WIMLIB_COMPRESSOR_OPS_H */
 
 #endif /* _WIMLIB_COMPRESSOR_OPS_H */

diff --git a/include/wimlib/decompressor_ops.h b/include/wimlib/decompressor_ops.h
index 44148b32b9e83102cb4b5e5e29640f00144ccccb..e51a671b826b7d690304558ec35b15c305fab349 100644 (file)
--- a/include/wimlib/decompressor_ops.h
+++ b/include/wimlib/decompressor_ops.h
@@ -25,5 +25,6 @@ struct decompressor_ops {
 extern const struct decompressor_ops lzx_decompressor_ops;
 extern const struct decompressor_ops xpress_decompressor_ops;
 extern const struct decompressor_ops lzms_decompressor_ops;
 extern const struct decompressor_ops lzx_decompressor_ops;
 extern const struct decompressor_ops xpress_decompressor_ops;
 extern const struct decompressor_ops lzms_decompressor_ops;

+extern const struct decompressor_ops zstd_decompressor_ops;

 
 #endif /* _WIMLIB_DECOMPRESSOR_OPS_H */
 
 #endif /* _WIMLIB_DECOMPRESSOR_OPS_H */

diff --git a/include/wimlib/header.h b/include/wimlib/header.h
index 894bd22591897491f2089ce75c4bd0dacb64939f..3228283541ea408d6be81443f7e07dafb39980f6 100644 (file)
--- a/include/wimlib/header.h
+++ b/include/wimlib/header.h
@@ -191,4 +191,6 @@ struct wim_header {
 /* XPRESS, with small chunk size???  */
 #define WIM_HDR_FLAG_COMPRESS_XPRESS_2 0x00200000
 
 /* XPRESS, with small chunk size???  */
 #define WIM_HDR_FLAG_COMPRESS_XPRESS_2 0x00200000
 

+#define WIM_HDR_FLAG_COMPRESS_ZSTD     0x00400000
+

 #endif /* _WIMLIB_HEADER_H */
 #endif /* _WIMLIB_HEADER_H */

diff --git a/programs/imagex.c b/programs/imagex.c
index a10ac488fdfbc4bb66949fdeae18a3193f253890..611934fa493283185232e7fc6bea662bc3183bf0 100644 (file)
--- a/programs/imagex.c
+++ b/programs/imagex.c
@@ -524,6 +524,9 @@ print_available_compression_types(FILE *fp)
        "    xpress (alias: \"fast\")\n"
        "    lzx    (alias: \"maximum\") (default for capture)\n"
        "    lzms   (alias: \"recovery\")\n"
        "    xpress (alias: \"fast\")\n"
        "    lzx    (alias: \"maximum\") (default for capture)\n"
        "    lzms   (alias: \"recovery\")\n"

+#ifdef WITH_ZSTD
+       "    zstd   (alias: \"zstandard\")\n"
+#endif

        "\n"
        );
        tfputs(s, fp);
        "\n"
        );
        tfputs(s, fp);


@@ -572,6 +575,11 @@ T(
                ctype = WIMLIB_COMPRESSION_TYPE_LZMS;
        } else if (!tstrcasecmp(optarg, T("lzms"))) {
                ctype = WIMLIB_COMPRESSION_TYPE_LZMS;
                ctype = WIMLIB_COMPRESSION_TYPE_LZMS;
        } else if (!tstrcasecmp(optarg, T("lzms"))) {
                ctype = WIMLIB_COMPRESSION_TYPE_LZMS;

+#ifdef WITH_ZSTD
+       } else if (!tstrcasecmp(optarg, T("zstd")) ||
+                  !tstrcasecmp(optarg, T("zstandard"))) {
+               ctype = WIMLIB_COMPRESSION_TYPE_ZSTD;
+#endif

        } else if (!tstrcasecmp(optarg, T("none"))) {
                ctype = WIMLIB_COMPRESSION_TYPE_NONE;
        } else {
        } else if (!tstrcasecmp(optarg, T("none"))) {
                ctype = WIMLIB_COMPRESSION_TYPE_NONE;
        } else {

diff --git a/src/compress.c b/src/compress.c
index 1b0d2c9f9aff5d0f04b1537bda5f264b3c9efd89..3c2b3e8a17a9ccf6e13fb0889d0573c2d5cc358b 100644 (file)
--- a/src/compress.c
+++ b/src/compress.c
@@ -42,6 +42,9 @@ static const struct compressor_ops * const compressor_ops[] = {
        [WIMLIB_COMPRESSION_TYPE_XPRESS] = &xpress_compressor_ops,
        [WIMLIB_COMPRESSION_TYPE_LZX]    = &lzx_compressor_ops,
        [WIMLIB_COMPRESSION_TYPE_LZMS]   = &lzms_compressor_ops,
        [WIMLIB_COMPRESSION_TYPE_XPRESS] = &xpress_compressor_ops,
        [WIMLIB_COMPRESSION_TYPE_LZX]    = &lzx_compressor_ops,
        [WIMLIB_COMPRESSION_TYPE_LZMS]   = &lzms_compressor_ops,

+#ifdef WITH_ZSTD
+       [WIMLIB_COMPRESSION_TYPE_ZSTD]   = &zstd_compressor_ops,
+#endif

 };
 
 /* Scale: 10 = low, 50 = medium, 100 = high */
 };
 
 /* Scale: 10 = low, 50 = medium, 100 = high */

diff --git a/src/decompress.c b/src/decompress.c
index f255ea92fd503906cd8a6d73480cde25b8c3adeb..7d8747a8b3506f76b2d5cdfc68ebe27abba4433d 100644 (file)
--- a/src/decompress.c
+++ b/src/decompress.c
@@ -40,6 +40,9 @@ static const struct decompressor_ops * const decompressor_ops[] = {
        [WIMLIB_COMPRESSION_TYPE_XPRESS] = &xpress_decompressor_ops,
        [WIMLIB_COMPRESSION_TYPE_LZX]    = &lzx_decompressor_ops,
        [WIMLIB_COMPRESSION_TYPE_LZMS]   = &lzms_decompressor_ops,
        [WIMLIB_COMPRESSION_TYPE_XPRESS] = &xpress_decompressor_ops,
        [WIMLIB_COMPRESSION_TYPE_LZX]    = &lzx_decompressor_ops,
        [WIMLIB_COMPRESSION_TYPE_LZMS]   = &lzms_decompressor_ops,

+#ifdef WITH_ZSTD
+       [WIMLIB_COMPRESSION_TYPE_ZSTD]   = &zstd_decompressor_ops,
+#endif

 };
 
 static bool
 };
 
 static bool

diff --git a/src/header.c b/src/header.c
index cf19396317ca3af6da227c6b4b60d1bd180b4b06..8b17160befdd351c767b9d18b00d84fc7ed991d1 100644 (file)
--- a/src/header.c
+++ b/src/header.c
@@ -213,6 +213,7 @@ static const struct {
        {WIM_HDR_FLAG_COMPRESS_XPRESS,  "COMPRESS_XPRESS"},
        {WIM_HDR_FLAG_COMPRESS_LZMS,    "COMPRESS_LZMS"},
        {WIM_HDR_FLAG_COMPRESS_XPRESS_2,"COMPRESS_XPRESS_2"},
        {WIM_HDR_FLAG_COMPRESS_XPRESS,  "COMPRESS_XPRESS"},
        {WIM_HDR_FLAG_COMPRESS_LZMS,    "COMPRESS_LZMS"},
        {WIM_HDR_FLAG_COMPRESS_XPRESS_2,"COMPRESS_XPRESS_2"},

+       {WIM_HDR_FLAG_COMPRESS_ZSTD,    "COMPRESS_ZSTD"},

 };
 
 /* API function documented in wimlib.h  */
 };
 
 /* API function documented in wimlib.h  */

diff --git a/src/wim.c b/src/wim.c
index ca40e53a47a303b4dbe3cb44e2172e2976e91da5..853a71e531236e03343407e1d1cf40452ade44f4 100644 (file)
--- a/src/wim.c
+++ b/src/wim.c
@@ -78,6 +78,15 @@ static const struct {
                .default_nonsolid_chunk_size = 131072,
                .default_solid_chunk_size = 67108864,
        },
                .default_nonsolid_chunk_size = 131072,
                .default_solid_chunk_size = 67108864,
        },

+#ifdef WITH_ZSTD
+       [WIMLIB_COMPRESSION_TYPE_ZSTD] = {
+               .name = T("Zstandard"),
+               .min_chunk_size = 4096,
+               .max_chunk_size = 268435456,
+               .default_nonsolid_chunk_size = 131072,
+               .default_solid_chunk_size = 131072,
+       },
+#endif

 };
 
 /* Is the specified compression type valid?  */
 };
 
 /* Is the specified compression type valid?  */


@@ -707,6 +716,10 @@ begin_read(WIMStruct *wim, const void *wim_filename_or_fd, int open_flags)
                        wim->compression_type = WIMLIB_COMPRESSION_TYPE_XPRESS;
                } else if (wim->hdr.flags & WIM_HDR_FLAG_COMPRESS_LZMS) {
                        wim->compression_type = WIMLIB_COMPRESSION_TYPE_LZMS;
                        wim->compression_type = WIMLIB_COMPRESSION_TYPE_XPRESS;
                } else if (wim->hdr.flags & WIM_HDR_FLAG_COMPRESS_LZMS) {
                        wim->compression_type = WIMLIB_COMPRESSION_TYPE_LZMS;

+#ifdef WITH_ZSTD
+               } else if (wim->hdr.flags & WIM_HDR_FLAG_COMPRESS_ZSTD) {
+                       wim->compression_type = WIMLIB_COMPRESSION_TYPE_ZSTD;
+#endif

                } else {
                        return WIMLIB_ERR_INVALID_COMPRESSION_TYPE;
                }
                } else {
                        return WIMLIB_ERR_INVALID_COMPRESSION_TYPE;
                }

diff --git a/src/write.c b/src/write.c
index 4805913de376fab95c1982eae45f0253a6dd13ec..835ad41bb341eff13a07a88fcf7172fb920dab73 100644 (file)
--- a/src/write.c
+++ b/src/write.c
@@ -2767,6 +2767,10 @@ write_wim_part(WIMStruct *wim,
                case WIMLIB_COMPRESSION_TYPE_LZMS:
                        wim->out_hdr.flags |= WIM_HDR_FLAG_COMPRESS_LZMS;
                        break;
                case WIMLIB_COMPRESSION_TYPE_LZMS:
                        wim->out_hdr.flags |= WIM_HDR_FLAG_COMPRESS_LZMS;
                        break;

+               case WIMLIB_COMPRESSION_TYPE_ZSTD:
+                       WARNING("Zstandard support is experimental -- use only for benchmarking!");
+                       wim->out_hdr.flags |= WIM_HDR_FLAG_COMPRESS_ZSTD;
+                       break;

                }
        }
 
                }
        }
 

diff --git a/src/zstd_compress.c b/src/zstd_compress.c
new file mode 100644 (file)
index 0000000..a5ece8b
--- /dev/null
+++ b/src/zstd_compress.c
@@ -0,0 +1,79 @@
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "wimlib/error.h"
+#include "wimlib/compressor_ops.h"
+#include "wimlib/util.h"
+
+#include <zstd.h>
+
+struct zstd_compressor {
+       unsigned compression_level;
+       ZSTD_CCtx *cctx;
+};
+
+static u64
+zstd_get_needed_memory(size_t max_bufsize, unsigned compression_level,
+                      bool destructive)
+{
+       return sizeof(struct zstd_compressor);
+}
+
+static int
+zstd_create_compressor(size_t max_bufsize, unsigned compression_level,
+                      bool destructive, void **c_ret)
+{
+       struct zstd_compressor *c;
+
+       c = MALLOC(sizeof(*c));
+       if (!c)
+               goto oom;
+
+       c->cctx = ZSTD_createCCtx();
+       if (!c->cctx)
+               goto oom;
+
+       c->compression_level = compression_level / 5;
+       c->compression_level = max(c->compression_level, 1);
+       c->compression_level = min(c->compression_level, ZSTD_maxCLevel());
+
+       *c_ret = c;
+       return 0;
+
+oom:
+       FREE(c);
+       return WIMLIB_ERR_NOMEM;
+}
+
+static size_t
+zstd_compress(const void *in, size_t in_nbytes,
+             void *out, size_t out_nbytes_avail, void *_c)
+{
+       struct zstd_compressor *c = _c;
+       size_t res;
+
+       res = ZSTD_compressCCtx(c->cctx, out, out_nbytes_avail, in, in_nbytes,
+                               c->compression_level);
+
+       if (ZSTD_isError(res))
+               return 0;
+       return res;
+}
+
+static void
+zstd_free_compressor(void *_c)
+{
+       struct zstd_compressor *c = _c;
+
+       FREE(c->cctx);
+       FREE(c);
+}
+
+const struct compressor_ops zstd_compressor_ops = {
+       .get_needed_memory      = zstd_get_needed_memory,
+       .create_compressor      = zstd_create_compressor,
+       .compress               = zstd_compress,
+       .free_compressor        = zstd_free_compressor,
+};
+

diff --git a/src/zstd_decompress.c b/src/zstd_decompress.c
new file mode 100644 (file)
index 0000000..d2b14a9
--- /dev/null
+++ b/src/zstd_decompress.c
@@ -0,0 +1,40 @@
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "wimlib/error.h"
+#include "wimlib/decompressor_ops.h"
+#include "wimlib/util.h"
+
+#include <zstd.h>
+
+static int
+zstd_create_decompressor(size_t max_block_size, void **d_ret)
+{
+       *d_ret = ZSTD_createDCtx();
+       if (!*d_ret)
+               return WIMLIB_ERR_NOMEM;
+       return 0;
+}
+
+static int
+zstd_decompress(const void *in, size_t in_nbytes,
+               void *out, size_t out_nbytes, void *_d)
+{
+       size_t res = ZSTD_decompressDCtx(_d, out, out_nbytes, in, in_nbytes);
+       if (res != out_nbytes)
+               return -1;
+       return 0;
+}
+
+static void
+zstd_free_decompressor(void *_d)
+{
+       ZSTD_freeDCtx(_d);
+}
+
+const struct decompressor_ops zstd_decompressor_ops = {
+       .create_decompressor    = zstd_create_decompressor,
+       .decompress             = zstd_decompress,
+       .free_decompressor      = zstd_free_decompressor,
+};