Resource fixes/changes
authorEric Biggers <ebiggers3@gmail.com>
Thu, 23 Aug 2012 03:11:32 +0000 (22:11 -0500)
committerEric Biggers <ebiggers3@gmail.com>
Thu, 23 Aug 2012 03:11:32 +0000 (22:11 -0500)
Fixes to the new WIM resource code

Also, write a resource uncompressed when the compressed resource is larger than
the uncompressed resource.

programs/imagex.c
src/lookup_table.h
src/modify.c
src/resource.c
src/sha1.c
src/sha1.h
src/wimlib.h
src/wimlib_internal.h
src/write.c

index a9bdab02e059454ceec778123940668f24322537..5a068528b83e889334a5868ceabb7e145a599555 100644 (file)
@@ -478,6 +478,7 @@ static int imagex_capture(int argc, const char **argv)
                        break;
                case 'v':
                        add_image_flags |= WIMLIB_ADD_IMAGE_FLAG_VERBOSE;
+                       write_flags |= WIMLIB_WRITE_FLAG_VERBOSE;
                        break;
                case 'N':
                        add_image_flags |= WIMLIB_ADD_IMAGE_FLAG_NTFS;
index 43dbd4bb9f250b1da8c3cf8c105c774d1619fc86..280376994176c36c8ef44a999902b72fe7253f7d 100644 (file)
@@ -123,6 +123,12 @@ static inline u64 wim_resource_size(const struct lookup_table_entry *lte)
        return lte->resource_entry.original_size;
 }
 
+static inline u64
+wim_resource_compressed_size(const struct lookup_table_entry *lte)
+{
+       return lte->resource_entry.size;
+}
+
 /*
  * XXX Probably should store the compression type directly in the lookup table
  * entry
index 5a89fd6b16026b397525e3cecf0b606ae55ff431..2807370e1a52dcf39dcda613ed02715242474b24 100644 (file)
@@ -86,6 +86,9 @@ static int build_dentry_tree(struct dentry *root, const char *root_disk_path,
        else
                stat_fn = lstat;
 
+       if (add_flags & WIMLIB_ADD_IMAGE_FLAG_VERBOSE)
+               printf("Scanning `%s'\n", root_disk_path);
+
 
        ret = (*stat_fn)(root_disk_path, &root_stbuf);
        if (ret != 0) {
index 8f85cba40b90f42c45c3bc288662fcbb4df2b124..f75ce7ab01a3f0a2d96b61a863706bff14caddb6 100644 (file)
@@ -346,8 +346,6 @@ err:
 int read_uncompressed_resource(FILE *fp, u64 offset, u64 len,
                               u8 contents_ret[])
 {
-       DEBUG("fp = %p, offset = %lu, len = %lu, contents_ret = %p",
-                       fp, offset, len, contents_ret);
        if (fseeko(fp, offset, SEEK_SET) != 0) {
                ERROR("Failed to seek to byte %"PRIu64" of input file "
                      "to read uncompressed resource (len = %"PRIu64")",
@@ -412,11 +410,10 @@ static int __read_wim_resource(const struct lookup_table_entry *lte,
 {
        /* We shouldn't be allowing read over-runs in any part of the library.
         * */
-       wimlib_assert(offset + size <= wim_resource_size(lte));
-
-       DEBUG("lte = %p, buf = %p, size = %zu, offset = %lu, raw = %d",
-                       lte, buf, size, offset, raw);
-       print_lookup_table_entry(lte);
+       if (raw)
+               wimlib_assert(offset + size <= lte->resource_entry.size);
+       else
+               wimlib_assert(offset + size <= lte->resource_entry.original_size);
 
        int ctype;
        int ret;
@@ -445,7 +442,7 @@ static int __read_wim_resource(const struct lookup_table_entry *lte,
 
                if (raw || ctype == WIM_COMPRESSION_TYPE_NONE)
                        return read_uncompressed_resource(lte->wim->fp,
-                                                         lte->resource_entry.offset,
+                                                         lte->resource_entry.offset + offset,
                                                          size, buf);
                else
                        return read_compressed_resource(lte->wim->fp,
@@ -521,8 +518,7 @@ int read_wim_resource(const struct lookup_table_entry *lte, u8 buf[],
  */
 int read_full_wim_resource(const struct lookup_table_entry *lte, u8 buf[])
 {
-       return __read_wim_resource(lte, buf, lte->resource_entry.original_size,
-                                  0, false);
+       return __read_wim_resource(lte, buf, wim_resource_size(lte), 0, false);
 }
 
 struct chunk_table {
@@ -548,6 +544,8 @@ begin_wim_resource_chunk_tab(const struct lookup_table_entry *lte,
                                               num_chunks * sizeof(u64));
        int ret = 0;
 
+       wimlib_assert(size != 0);
+
        if (!chunk_tab) {
                ERROR("Failed to allocate chunk table for %"PRIu64" byte "
                      "resource", size);
@@ -556,17 +554,15 @@ begin_wim_resource_chunk_tab(const struct lookup_table_entry *lte,
        }
        chunk_tab->file_offset = file_offset;
        chunk_tab->num_chunks = num_chunks;
-       chunk_tab->cur_offset_p = chunk_tab->offsets;
-       chunk_tab->original_resource_size = lte->resource_entry.original_size;
-       chunk_tab->bytes_per_chunk_entry =
-                       (lte->resource_entry.original_size >= (1ULL << 32))
-                                ? 8 : 4;
+       chunk_tab->original_resource_size = size;
+       chunk_tab->bytes_per_chunk_entry = (size >= (1ULL << 32)) ? 8 : 4;
        chunk_tab->table_disk_size = chunk_tab->bytes_per_chunk_entry *
                                     (num_chunks - 1);
+       chunk_tab->cur_offset = 0;
+       chunk_tab->cur_offset_p = chunk_tab->offsets;
 
        if (fwrite(chunk_tab, 1, chunk_tab->table_disk_size, out_fp) !=
-                  chunk_tab->table_disk_size)
-       {
+                  chunk_tab->table_disk_size) {
                ERROR_WITH_ERRNO("Failed to write chunk table in compressed "
                                 "file resource");
                ret = WIMLIB_ERR_WRITE;
@@ -578,18 +574,25 @@ out:
        return ret;
 }
 
-static int compress_chunk(const u8 chunk[], unsigned chunk_sz,
+static int compress_chunk(const u8 chunk[], unsigned chunk_size,
                          u8 compressed_chunk[],
                          unsigned *compressed_chunk_len_ret,
                          int ctype)
 {
        unsigned compressed_chunk_sz;
        int (*compress)(const void *, unsigned, void *, unsigned *);
-       if (ctype == WIM_COMPRESSION_TYPE_LZX)
+       switch (ctype) {
+       case WIM_COMPRESSION_TYPE_LZX:
                compress = lzx_compress;
-       else
+               break;
+       case WIM_COMPRESSION_TYPE_XPRESS:
                compress = xpress_compress;
-       return (*compress)(chunk, chunk_sz, compressed_chunk,
+               break;
+       default:
+               wimlib_assert(0);
+               break;
+       }
+       return (*compress)(chunk, chunk_size, compressed_chunk,
                           compressed_chunk_len_ret);
 }
 
@@ -601,6 +604,7 @@ static int write_wim_resource_chunk(const u8 chunk[], unsigned chunk_size,
        unsigned out_chunk_size;
 
        if (out_ctype == WIM_COMPRESSION_TYPE_NONE) {
+               wimlib_assert(chunk_tab == NULL);
                out_chunk = chunk;
                out_chunk_size = chunk_size;
        } else {
@@ -612,13 +616,11 @@ static int write_wim_resource_chunk(const u8 chunk[], unsigned chunk_size,
 
                ret = compress_chunk(chunk, chunk_size, compressed_chunk,
                                     &out_chunk_size, out_ctype);
-               if (ret > 0)
-                       return ret;
-               else if (ret < 0) {
+               if (ret == 0) {
+                       out_chunk = compressed_chunk;
+               } else {
                        out_chunk = chunk;
                        out_chunk_size = chunk_size;
-               } else {
-                       out_chunk = compressed_chunk;
                }
                *chunk_tab->cur_offset_p++ = chunk_tab->cur_offset;
                chunk_tab->cur_offset += out_chunk_size;
@@ -635,32 +637,33 @@ static int
 finish_wim_resource_chunk_tab(struct chunk_table *chunk_tab,
                              FILE *out_fp, u64 *compressed_size_p)
 {
+       size_t bytes_written;
        if (fseeko(out_fp, chunk_tab->file_offset, SEEK_SET) != 0) {
-               ERROR_WITH_ERRNO("Failed to seet to byte "PRIu64" of output "
+               ERROR_WITH_ERRNO("Failed to seek to byte "PRIu64" of output "
                                 "WIM file", chunk_tab->file_offset);
                return WIMLIB_ERR_WRITE;
        }
 
        if (chunk_tab->bytes_per_chunk_entry == 8) {
-               array_to_le64(chunk_tab->offsets, chunk_tab->num_chunks - 1);
+               array_to_le64(chunk_tab->offsets, chunk_tab->num_chunks);
        } else {
-               for (u64 i = 0; i < chunk_tab->num_chunks - 1; i++)
+               for (u64 i = 0; i < chunk_tab->num_chunks; i++)
                        ((u32*)chunk_tab->offsets)[i] =
                                to_le32(chunk_tab->offsets[i]);
        }
-       if (fwrite(chunk_tab->offsets, 1, chunk_tab->table_disk_size, out_fp) !=
-                  chunk_tab->table_disk_size)
-       {
+       bytes_written = fwrite((u8*)chunk_tab->offsets +
+                                       chunk_tab->bytes_per_chunk_entry,
+                              1, chunk_tab->table_disk_size, out_fp);
+       if (bytes_written != chunk_tab->table_disk_size) {
                ERROR_WITH_ERRNO("Failed to write chunk table in compressed "
                                 "file resource");
                return WIMLIB_ERR_WRITE;
        }
-       if (fseeko(out_fp, chunk_tab->file_offset, SEEK_SET) != 0) {
-               ERROR_WITH_ERRNO("Failed to seet to byte "PRIu64" of output "
-                                "WIM file", chunk_tab->file_offset);
+       if (fseeko(out_fp, 0, SEEK_END) != 0) {
+               ERROR_WITH_ERRNO("Failed to seek to end of output WIM file");
                return WIMLIB_ERR_WRITE;
        }
-       *compressed_size_p = chunk_tab->cur_offset;
+       *compressed_size_p = chunk_tab->cur_offset + chunk_tab->table_disk_size;
        return 0;
 }
 
@@ -680,79 +683,126 @@ finish_wim_resource_chunk_tab(struct chunk_table *chunk_tab,
  *
  * Returns 0 on success; nonzero on failure.
  */
-static int __write_wim_resource(const struct lookup_table_entry *lte,
-                               FILE *out_fp, int out_ctype,
-                               struct resource_entry *out_res_entry)
+static int write_wim_resource(struct lookup_table_entry *lte,
+                             FILE *out_fp, int out_ctype,
+                             struct resource_entry *out_res_entry)
 {
-       u64 size = wim_resource_size(lte);
-       u64 bytes_remaining = size;
-       char buf[min(WIM_CHUNK_SIZE, bytes_remaining)];
+       u64 bytes_remaining;
+       u64 original_size;
+       u64 old_compressed_size;
+       u64 new_compressed_size;
        u64 offset = 0;
-       u64 compressed_size = bytes_remaining;
        int ret = 0;
        struct chunk_table *chunk_tab = NULL;
        bool raw;
        off_t file_offset;
 
-       if (out_res_entry) {
-               file_offset = ftello(out_fp);
-               if (file_offset == -1) {
-                       ERROR_WITH_ERRNO("Failed to get offset in output "
-                                        "stream");
-                       return WIMLIB_ERR_WRITE;
-               }
+       original_size = wim_resource_size(lte);
+       old_compressed_size = wim_resource_compressed_size(lte);
+
+       file_offset = ftello(out_fp);
+       if (file_offset == -1) {
+               ERROR_WITH_ERRNO("Failed to get offset in output "
+                                "stream");
+               return WIMLIB_ERR_WRITE;
        }
        
        raw = (wim_resource_compression_type(lte) == out_ctype);
        if (raw)
-               out_ctype = WIM_COMPRESSION_TYPE_NONE;
+               bytes_remaining = old_compressed_size;
+       else
+               bytes_remaining = original_size;
+
+       if (bytes_remaining == 0)
+               return 0;
+
+       char buf[min(WIM_CHUNK_SIZE, bytes_remaining)];
 
-       if (out_ctype != WIM_COMPRESSION_TYPE_NONE) {
+       if (out_ctype != WIM_COMPRESSION_TYPE_NONE && !raw) {
                ret = begin_wim_resource_chunk_tab(lte, out_fp, file_offset,
                                                   &chunk_tab);
                if (ret != 0)
                        goto out;
        }
+       if (lte->resource_location == RESOURCE_IN_FILE_ON_DISK
+            && !lte->file_on_disk_fp)
+       {
+               wimlib_assert(lte->file_on_disk);
+               lte->file_on_disk_fp = fopen(lte->file_on_disk, "rb");
+               if (!lte->file_on_disk_fp) {
+                       ERROR_WITH_ERRNO("Failed to open the file `%s' for "
+                                        "reading", lte->file_on_disk);
+                       ret = WIMLIB_ERR_OPEN;
+                       goto out;
+               }
+       }
 
-       while (bytes_remaining) {
+       do {
                u64 to_read = min(bytes_remaining, WIM_CHUNK_SIZE);
                ret = __read_wim_resource(lte, buf, to_read, offset, raw);
                if (ret != 0)
-                       goto out;
+                       goto out_fclose;
                ret = write_wim_resource_chunk(buf, to_read, out_fp,
                                               out_ctype, chunk_tab);
                if (ret != 0)
-                       goto out;
+                       goto out_fclose;
                bytes_remaining -= to_read;
                offset += to_read;
-       }
-       if (out_ctype != WIM_COMPRESSION_TYPE_NONE) {
+       } while (bytes_remaining);
+       if (out_ctype != WIM_COMPRESSION_TYPE_NONE && !raw) {
                ret = finish_wim_resource_chunk_tab(chunk_tab, out_fp,
-                                                   &compressed_size);
+                                                   &new_compressed_size);
                if (ret != 0)
-                       goto out;
+                       goto out_fclose;
+       } else {
+               new_compressed_size = old_compressed_size;
+       }
+
+       if (new_compressed_size > original_size) {
+               /* Oops!  We compressed the resource to larger than the original
+                * size.  Write the resource uncompressed instead. */
+               if (fseeko(out_fp, file_offset, SEEK_SET) != 0) {
+                       ERROR_WITH_ERRNO("Failed to seek to byte "PRIu64" "
+                                        "of output WIM file", file_offset);
+                       ret = WIMLIB_ERR_WRITE;
+                       goto out_fclose;
+               }
+               ret = write_wim_resource(lte, out_fp, WIM_COMPRESSION_TYPE_NONE,
+                                        out_res_entry);
+               if (ret != 0)
+                       goto out_fclose;
+               if (fflush(out_fp) != 0) {
+                       ERROR_WITH_ERRNO("Failed to flush output WIM file");
+                       ret = WIMLIB_ERR_WRITE;
+                       goto out_fclose;
+               }
+               if (ftruncate(fileno(out_fp), file_offset + out_res_entry->size) != 0) {
+                       ERROR_WITH_ERRNO("Failed to truncate output WIM file");
+                       ret = WIMLIB_ERR_WRITE;
+               }
+               goto out_fclose;
        }
+       wimlib_assert(new_compressed_size <= original_size);
        if (out_res_entry) {
-               out_res_entry->size          = compressed_size;
-               out_res_entry->original_size = size;
-               out_res_entry->offset       = file_offset;
+               out_res_entry->size          = new_compressed_size;
+               out_res_entry->original_size = original_size;
+               out_res_entry->offset        = file_offset;
                if (out_ctype == WIM_COMPRESSION_TYPE_NONE)
                        out_res_entry->flags = 0;
                else
                        out_res_entry->flags = WIM_RESHDR_FLAG_COMPRESSED;
        }
+out_fclose:
+       if (lte->resource_location == RESOURCE_IN_FILE_ON_DISK
+            && lte->file_on_disk_fp) {
+               fclose(lte->file_on_disk_fp);
+               lte->file_on_disk_fp = NULL;
+       }
 out:
        FREE(chunk_tab);
        return ret;
 }
 
-static int write_wim_resource(struct lookup_table_entry *lte,
-                             FILE *out_fp, int out_ctype)
-{
-       return __write_wim_resource(lte, out_fp, out_ctype,
-                                   &lte->output_resource_entry);
-}
-
 static int write_wim_resource_from_buffer(const u8 *buf, u64 buf_size,
                                          FILE *out_fp, int out_ctype,
                                          struct resource_entry *out_res_entry)
@@ -764,7 +814,7 @@ static int write_wim_resource_from_buffer(const u8 *buf, u64 buf_size,
        lte.resource_entry.offset = 0;
        lte.resource_location = RESOURCE_IN_ATTACHED_BUFFER;
        lte.attached_buffer = (u8*)buf;
-       return __write_wim_resource(&lte, out_fp, out_ctype, out_res_entry);
+       return write_wim_resource(&lte, out_fp, out_ctype, out_res_entry);
 }
 
 /* 
@@ -796,8 +846,7 @@ int extract_wim_resource_to_fd(const struct lookup_table_entry *lte, int fd,
 
 int extract_full_wim_resource_to_fd(const struct lookup_table_entry *lte, int fd)
 {
-       return extract_wim_resource_to_fd(lte, fd,
-                                         lte->resource_entry.original_size);
+       return extract_wim_resource_to_fd(lte, fd, wim_resource_size(lte));
 }
 
 /* 
@@ -820,7 +869,8 @@ int copy_resource(struct lookup_table_entry *lte, void *wim)
            !w->write_metadata)
                return 0;
 
-       ret = write_wim_resource(lte, w->out_fp, wimlib_get_compression_type(w));
+       ret = write_wim_resource(lte, w->out_fp, wimlib_get_compression_type(w), 
+                                &lte->output_resource_entry);
        if (ret != 0)
                return ret;
        lte->out_refcnt = lte->refcnt;
@@ -846,10 +896,16 @@ int write_dentry_resources(struct dentry *dentry, void *wim_p)
        struct lookup_table_entry *lte;
        int ctype = wimlib_get_compression_type(w);
 
+       if (w->write_flags & WIMLIB_WRITE_FLAG_VERBOSE) {
+               wimlib_assert(dentry->full_path_utf8);
+               printf("Writing streams for `%s'\n", dentry->full_path_utf8);
+       }
+
        for (unsigned i = 0; i <= dentry->num_ads; i++) {
                lte = dentry_stream_lte(dentry, i, w->lookup_table);
                if (lte && ++lte->out_refcnt == 1) {
-                       ret = write_wim_resource(lte, w->out_fp, ctype);
+                       ret = write_wim_resource(lte, w->out_fp, ctype,
+                                                &lte->output_resource_entry);
                        if (ret != 0)
                                break;
                }
index 7fd67bce992c86006dc9f05eaf56a6236293dfdc..36ff3f150bbc4f637dcbe33054c690347ff2108b 100644 (file)
  * Steve Reid's public domain code, or based on Intel's SSSE3 SHA1 code.
  */
 
-const u8 empty_file_sha1sum[SHA1_HASH_SIZE] = {
-       0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, 
-       0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07, 0x09,
-};
-
-
 #ifdef WITH_LIBCRYPTO
 
 #define sha1_init     SHA1_Init
index f668116bc37944ca553fa06a0991f121bf16805d..db524e6de60d5a579804ace47682423e73dc9d60 100644 (file)
@@ -9,13 +9,6 @@
 
 #define SHA1_HASH_SIZE 20
 
-extern const u8 empty_file_sha1sum[SHA1_HASH_SIZE];
-
-static inline bool is_empty_file_hash(const u8 hash[SHA1_HASH_SIZE])
-{
-       return memcmp(hash, empty_file_sha1sum, SHA1_HASH_SIZE) == 0;
-}
-
 static inline void copy_hash(u8 dest[SHA1_HASH_SIZE],
                             const u8 src[SHA1_HASH_SIZE])
 {
index ba787c32539d1c911f8ed8feefdea9ee7a1de4d4..822c2c81163e3de763023e40ed4616446061e138 100644 (file)
@@ -264,6 +264,9 @@ enum wim_compression_type {
 /** Print progress information when writing the integrity table. */
 #define WIMLIB_WRITE_FLAG_SHOW_PROGRESS                0x00000002
 
+/** Print file paths as we write then */
+#define WIMLIB_WRITE_FLAG_VERBOSE              0x00000004
+
 /** Mark the image being added as the bootable image of the WIM. */
 #define WIMLIB_ADD_IMAGE_FLAG_BOOT             0x00000001
 
index 4271cd492b122a3b456eb521081eb1da2f59228a..d13b46e562364f27c4819f8d82ef81e4f7bfe585 100644 (file)
@@ -233,19 +233,19 @@ struct image_metadata {
 typedef struct WIMStruct {
 
        /* A pointer to the file indicated by @filename, opened for reading. */
-       FILE                *fp;
+       FILE  *fp;
 
        /* FILE pointer for the WIM file that is being written. */
        FILE  *out_fp;
 
        /* The name of the WIM file that has been opened. */
-       char                *filename;
+       char  *filename;
 
        /* The lookup table for the WIM file. */ 
        struct lookup_table *lookup_table;
 
        /* Pointer to the XML data read from the WIM file. */
-       u8                  *xml_data;
+       u8    *xml_data;
 
        /* Information retrieved from the XML data, arranged
         * in an orderly manner. */
@@ -263,6 +263,7 @@ typedef struct WIMStruct {
        union {
                int extract_flags;
                int add_flags;
+               int write_flags;
        };
 
        /* The currently selected image, indexed starting at 1.  If not 0,
index b4aa2655207d819d30799a879d4de1a93cb46c71..9aa8116c0b3dbb8b84b2c22e3c08eb3d85a04890 100644 (file)
@@ -416,6 +416,8 @@ WIMLIBAPI int wimlib_write(WIMStruct *w, const char *path, int image, int flags)
 
        for_lookup_table_entry(w->lookup_table, zero_out_refcnts, NULL);
 
+       w->write_flags = flags;
+
        ret = for_image(w, image, write_file_resources);
        if (ret != 0) {
                ERROR("Failed to write WIM file resources to `%s'", path);