[EXPERIMENTAL, FOR BENCHMARKING ONLY] Zstandard compression support zstd_support
authorEric Biggers <ebiggers3@gmail.com>
Wed, 2 May 2018 03:40:55 +0000 (20:40 -0700)
committerEric Biggers <ebiggers3@gmail.com>
Wed, 2 May 2018 04:17:05 +0000 (21:17 -0700)
14 files changed:
Makefile.am
configure.ac
include/wimlib.h
include/wimlib/compressor_ops.h
include/wimlib/decompressor_ops.h
include/wimlib/header.h
programs/imagex.c
src/compress.c
src/decompress.c
src/header.c
src/wim.c
src/write.c
src/zstd_compress.c [new file with mode: 0644]
src/zstd_decompress.c [new file with mode: 0644]

index 603c8d9..784dfc0 100644 (file)
@@ -164,6 +164,11 @@ libwim_la_SOURCES += src/ntfs-3g_apply.c   \
                     include/wimlib/ntfs_3g.h
 endif
 
+if WITH_ZSTD
+libwim_la_SOURCES += src/zstd_compress.c               \
+                    src/zstd_decompress.c
+endif
+
 if WINDOWS_NATIVE_BUILD
 libwim_la_SOURCES += src/wimboot.c                     \
                     src/win32_common.c                 \
@@ -199,6 +204,7 @@ libwim_la_CFLAGS =          \
        $(LIBXML2_CFLAGS)       \
        $(LIBNTFS_3G_CFLAGS)    \
        $(LIBFUSE_CFLAGS)       \
+       $(LIBZSTD_CFLAGS)       \
        $(LIBCRYPTO_CFLAGS)
 
 libwim_la_LDFLAGS = $(AM_LDFLAGS) -version-info 29:0:14
@@ -209,6 +215,7 @@ libwim_la_LIBADD =          \
        $(LIBNTFS_3G_LIBS)      \
        $(LIBFUSE_LIBS)         \
        $(LIBRT_LIBS)           \
+       $(LIBZSTD_LIBS)         \
        $(LIBCRYPTO_LIBS)       \
        $(PLATFORM_LIBS)
 
index 072da7d..57dd78b 100644 (file)
@@ -221,6 +221,27 @@ else
 fi
 AM_CONDITIONAL([ENABLE_SSSE3_SHA1], [test "$ENABLE_SSSE3_SHA1" = "yes"])
 
+# --------------------------- Zstandard support -------------------------------
+
+AC_MSG_CHECKING([whether to include support for Zstandard compression])
+AC_ARG_WITH([zstd],
+           [AS_HELP_STRING([--with-zstd],
+                           [Zstandard compression support via libzstd
+                            (EXPERIMENTAL)])],
+           [WITH_ZSTD=$withval],
+           [WITH_ZSTD=no])
+AC_MSG_RESULT([$WITH_ZSTD])
+
+if test "$WITH_ZSTD" = "yes" ; then
+       PKG_CHECK_MODULES([LIBZSTD], [libzstd], [],
+               [AC_MSG_ERROR([Cannot find libzstd!  Either install libzstd, or
+                configure --without-zstd to disable Zstandard compression
+                support.])])
+       PKGCONFIG_PRIVATE_REQUIRES="$PKGCONFIG_PRIVATE_REQUIRES zstd"
+       AC_DEFINE([WITH_ZSTD], [1], [Define to 1 if using Zstandard support])
+fi
+AM_CONDITIONAL([WITH_ZSTD], [test "$WITH_ZSTD" = "yes"])
+
 # ----------------------------- Other options ---------------------------------
 
 AC_MSG_CHECKING([whether to include error messages])
index 047716c..8a1d71b 100644 (file)
@@ -568,6 +568,18 @@ enum wimlib_compression_type {
         * up to and including <c>2^30</c>.
         */
        WIMLIB_COMPRESSION_TYPE_LZMS = 3,
+
+       /**
+        * The Zstandard compression format.  This is a wimlib extension;
+        * Microsoft's WIMGAPI does not support this format.  Zstandard supports
+        * chunk sizes from 4096 to 268435456 bytes, with the default being
+        * 131072.  Multiple compression levels are supported as well.  For
+        * consistency with wimlib's other compression types, the compression
+        * levels exposed by libzstd are multiplied by 5; e.g., wimlib's level
+        * 50 (the default level) corresponds to libzstd's level 10.  libzstd
+        * allows levels 1 through 22.
+        */
+       WIMLIB_COMPRESSION_TYPE_ZSTD = 4,
 };
 
 /** @} */
index bef5cf6..c8df2fb 100644 (file)
@@ -32,5 +32,6 @@ struct compressor_ops {
 extern const struct compressor_ops lzx_compressor_ops;
 extern const struct compressor_ops xpress_compressor_ops;
 extern const struct compressor_ops lzms_compressor_ops;
+extern const struct compressor_ops zstd_compressor_ops;
 
 #endif /* _WIMLIB_COMPRESSOR_OPS_H */
index 44148b3..e51a671 100644 (file)
@@ -25,5 +25,6 @@ struct decompressor_ops {
 extern const struct decompressor_ops lzx_decompressor_ops;
 extern const struct decompressor_ops xpress_decompressor_ops;
 extern const struct decompressor_ops lzms_decompressor_ops;
+extern const struct decompressor_ops zstd_decompressor_ops;
 
 #endif /* _WIMLIB_DECOMPRESSOR_OPS_H */
index 894bd22..3228283 100644 (file)
@@ -191,4 +191,6 @@ struct wim_header {
 /* XPRESS, with small chunk size???  */
 #define WIM_HDR_FLAG_COMPRESS_XPRESS_2 0x00200000
 
+#define WIM_HDR_FLAG_COMPRESS_ZSTD     0x00400000
+
 #endif /* _WIMLIB_HEADER_H */
index a10ac48..611934f 100644 (file)
@@ -524,6 +524,9 @@ print_available_compression_types(FILE *fp)
        "    xpress (alias: \"fast\")\n"
        "    lzx    (alias: \"maximum\") (default for capture)\n"
        "    lzms   (alias: \"recovery\")\n"
+#ifdef WITH_ZSTD
+       "    zstd   (alias: \"zstandard\")\n"
+#endif
        "\n"
        );
        tfputs(s, fp);
@@ -572,6 +575,11 @@ T(
                ctype = WIMLIB_COMPRESSION_TYPE_LZMS;
        } else if (!tstrcasecmp(optarg, T("lzms"))) {
                ctype = WIMLIB_COMPRESSION_TYPE_LZMS;
+#ifdef WITH_ZSTD
+       } else if (!tstrcasecmp(optarg, T("zstd")) ||
+                  !tstrcasecmp(optarg, T("zstandard"))) {
+               ctype = WIMLIB_COMPRESSION_TYPE_ZSTD;
+#endif
        } else if (!tstrcasecmp(optarg, T("none"))) {
                ctype = WIMLIB_COMPRESSION_TYPE_NONE;
        } else {
index 1b0d2c9..3c2b3e8 100644 (file)
@@ -42,6 +42,9 @@ static const struct compressor_ops * const compressor_ops[] = {
        [WIMLIB_COMPRESSION_TYPE_XPRESS] = &xpress_compressor_ops,
        [WIMLIB_COMPRESSION_TYPE_LZX]    = &lzx_compressor_ops,
        [WIMLIB_COMPRESSION_TYPE_LZMS]   = &lzms_compressor_ops,
+#ifdef WITH_ZSTD
+       [WIMLIB_COMPRESSION_TYPE_ZSTD]   = &zstd_compressor_ops,
+#endif
 };
 
 /* Scale: 10 = low, 50 = medium, 100 = high */
index f255ea9..7d8747a 100644 (file)
@@ -40,6 +40,9 @@ static const struct decompressor_ops * const decompressor_ops[] = {
        [WIMLIB_COMPRESSION_TYPE_XPRESS] = &xpress_decompressor_ops,
        [WIMLIB_COMPRESSION_TYPE_LZX]    = &lzx_decompressor_ops,
        [WIMLIB_COMPRESSION_TYPE_LZMS]   = &lzms_decompressor_ops,
+#ifdef WITH_ZSTD
+       [WIMLIB_COMPRESSION_TYPE_ZSTD]   = &zstd_decompressor_ops,
+#endif
 };
 
 static bool
index cf19396..8b17160 100644 (file)
@@ -213,6 +213,7 @@ static const struct {
        {WIM_HDR_FLAG_COMPRESS_XPRESS,  "COMPRESS_XPRESS"},
        {WIM_HDR_FLAG_COMPRESS_LZMS,    "COMPRESS_LZMS"},
        {WIM_HDR_FLAG_COMPRESS_XPRESS_2,"COMPRESS_XPRESS_2"},
+       {WIM_HDR_FLAG_COMPRESS_ZSTD,    "COMPRESS_ZSTD"},
 };
 
 /* API function documented in wimlib.h  */
index ca40e53..853a71e 100644 (file)
--- a/src/wim.c
+++ b/src/wim.c
@@ -78,6 +78,15 @@ static const struct {
                .default_nonsolid_chunk_size = 131072,
                .default_solid_chunk_size = 67108864,
        },
+#ifdef WITH_ZSTD
+       [WIMLIB_COMPRESSION_TYPE_ZSTD] = {
+               .name = T("Zstandard"),
+               .min_chunk_size = 4096,
+               .max_chunk_size = 268435456,
+               .default_nonsolid_chunk_size = 131072,
+               .default_solid_chunk_size = 131072,
+       },
+#endif
 };
 
 /* Is the specified compression type valid?  */
@@ -707,6 +716,10 @@ begin_read(WIMStruct *wim, const void *wim_filename_or_fd, int open_flags)
                        wim->compression_type = WIMLIB_COMPRESSION_TYPE_XPRESS;
                } else if (wim->hdr.flags & WIM_HDR_FLAG_COMPRESS_LZMS) {
                        wim->compression_type = WIMLIB_COMPRESSION_TYPE_LZMS;
+#ifdef WITH_ZSTD
+               } else if (wim->hdr.flags & WIM_HDR_FLAG_COMPRESS_ZSTD) {
+                       wim->compression_type = WIMLIB_COMPRESSION_TYPE_ZSTD;
+#endif
                } else {
                        return WIMLIB_ERR_INVALID_COMPRESSION_TYPE;
                }
index 4805913..835ad41 100644 (file)
@@ -2767,6 +2767,10 @@ write_wim_part(WIMStruct *wim,
                case WIMLIB_COMPRESSION_TYPE_LZMS:
                        wim->out_hdr.flags |= WIM_HDR_FLAG_COMPRESS_LZMS;
                        break;
+               case WIMLIB_COMPRESSION_TYPE_ZSTD:
+                       WARNING("Zstandard support is experimental -- use only for benchmarking!");
+                       wim->out_hdr.flags |= WIM_HDR_FLAG_COMPRESS_ZSTD;
+                       break;
                }
        }
 
diff --git a/src/zstd_compress.c b/src/zstd_compress.c
new file mode 100644 (file)
index 0000000..a5ece8b
--- /dev/null
@@ -0,0 +1,79 @@
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "wimlib/error.h"
+#include "wimlib/compressor_ops.h"
+#include "wimlib/util.h"
+
+#include <zstd.h>
+
+struct zstd_compressor {
+       unsigned compression_level;
+       ZSTD_CCtx *cctx;
+};
+
+static u64
+zstd_get_needed_memory(size_t max_bufsize, unsigned compression_level,
+                      bool destructive)
+{
+       return sizeof(struct zstd_compressor);
+}
+
+static int
+zstd_create_compressor(size_t max_bufsize, unsigned compression_level,
+                      bool destructive, void **c_ret)
+{
+       struct zstd_compressor *c;
+
+       c = MALLOC(sizeof(*c));
+       if (!c)
+               goto oom;
+
+       c->cctx = ZSTD_createCCtx();
+       if (!c->cctx)
+               goto oom;
+
+       c->compression_level = compression_level / 5;
+       c->compression_level = max(c->compression_level, 1);
+       c->compression_level = min(c->compression_level, ZSTD_maxCLevel());
+
+       *c_ret = c;
+       return 0;
+
+oom:
+       FREE(c);
+       return WIMLIB_ERR_NOMEM;
+}
+
+static size_t
+zstd_compress(const void *in, size_t in_nbytes,
+             void *out, size_t out_nbytes_avail, void *_c)
+{
+       struct zstd_compressor *c = _c;
+       size_t res;
+
+       res = ZSTD_compressCCtx(c->cctx, out, out_nbytes_avail, in, in_nbytes,
+                               c->compression_level);
+
+       if (ZSTD_isError(res))
+               return 0;
+       return res;
+}
+
+static void
+zstd_free_compressor(void *_c)
+{
+       struct zstd_compressor *c = _c;
+
+       FREE(c->cctx);
+       FREE(c);
+}
+
+const struct compressor_ops zstd_compressor_ops = {
+       .get_needed_memory      = zstd_get_needed_memory,
+       .create_compressor      = zstd_create_compressor,
+       .compress               = zstd_compress,
+       .free_compressor        = zstd_free_compressor,
+};
+
diff --git a/src/zstd_decompress.c b/src/zstd_decompress.c
new file mode 100644 (file)
index 0000000..d2b14a9
--- /dev/null
@@ -0,0 +1,40 @@
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "wimlib/error.h"
+#include "wimlib/decompressor_ops.h"
+#include "wimlib/util.h"
+
+#include <zstd.h>
+
+static int
+zstd_create_decompressor(size_t max_block_size, void **d_ret)
+{
+       *d_ret = ZSTD_createDCtx();
+       if (!*d_ret)
+               return WIMLIB_ERR_NOMEM;
+       return 0;
+}
+
+static int
+zstd_decompress(const void *in, size_t in_nbytes,
+               void *out, size_t out_nbytes, void *_d)
+{
+       size_t res = ZSTD_decompressDCtx(_d, out, out_nbytes, in, in_nbytes);
+       if (res != out_nbytes)
+               return -1;
+       return 0;
+}
+
+static void
+zstd_free_decompressor(void *_d)
+{
+       ZSTD_freeDCtx(_d);
+}
+
+const struct decompressor_ops zstd_decompressor_ops = {
+       .create_decompressor    = zstd_create_decompressor,
+       .decompress             = zstd_decompress,
+       .free_decompressor      = zstd_free_decompressor,
+};