Get append working

diff --git a/src/lzx-compress.c b/src/lzx-compress.c
index a8e15b24f107cf7239bb24d5416399574e725d99..07fc94724bd0fd73489b3973d6c6a54f0db7b960 100644 (file)
--- a/src/lzx-compress.c
+++ b/src/lzx-compress.c
@@ -669,6 +669,7 @@ wimlib_lzx_compress(const void *__uncompressed_data, unsigned uncompressed_len,
        /* The input data must be preprocessed. To avoid changing the original
         * input, copy it to a temporary buffer. */
        memcpy(uncompressed_data, __uncompressed_data, uncompressed_len);
        /* The input data must be preprocessed. To avoid changing the original
         * input, copy it to a temporary buffer. */
        memcpy(uncompressed_data, __uncompressed_data, uncompressed_len);

+       memset(uncompressed_data + uncompressed_len, 0, 8);

 
        /* Before doing any actual compression, do the call instruction (0xe8
         * byte) translation on the uncompressed data. */
 
        /* Before doing any actual compression, do the call instruction (0xe8
         * byte) translation on the uncompressed data. */

diff --git a/src/wimlib.h b/src/wimlib.h
index ea5966ce9bc23c5060a07295dfe00b921b9fa687..974fba7f61529d2ef0af86ac5e9362375c6d1573 100644 (file)
--- a/src/wimlib.h
+++ b/src/wimlib.h
@@ -1674,14 +1674,6 @@ wimlib_join(const wimlib_tchar * const *swms,
  *
  * As a special requirement, the compression code is optimized for the WIM
  * format and therefore requires (@a chunk_size <= 32768).
  *
  * As a special requirement, the compression code is optimized for the WIM
  * format and therefore requires (@a chunk_size <= 32768).

- *
- * As another special requirement, the compression code will read up to 8 bytes
- * off the end of the @a chunk array for performance reasons.  The values of
- * these bytes will not affect the output of the compression, but the calling
- * code must make sure that the buffer holding the uncompressed chunk is
- * actually at least (@a chunk_size + 8) bytes, or at least that these extra
- * bytes are in mapped memory that will not cause a memory access violation if
- * accessed.

  */
 extern unsigned
 wimlib_lzx_compress(const void *chunk, unsigned chunk_size, void *out);
  */
 extern unsigned
 wimlib_lzx_compress(const void *chunk, unsigned chunk_size, void *out);

diff --git a/src/write.c b/src/write.c
index ccad7f07caa5c3f4b7af309995ab5974d2742aa0..1891c89b20685a0caf05b3ce3bb8cb6755c51566 100644 (file)
--- a/src/write.c
+++ b/src/write.c
@@ -656,15 +656,14 @@ do_write_stream_list(struct list_head *my_resources,
                                duplicate_lte->out_refcnt += lte->refcnt;
                                *my_ptr = duplicate_lte;
                                free_lookup_table_entry(lte);
                                duplicate_lte->out_refcnt += lte->refcnt;
                                *my_ptr = duplicate_lte;
                                free_lookup_table_entry(lte);

-
+                               lte = duplicate_lte;

                                if (new_stream) {
                                if (new_stream) {

-                                       lte = duplicate_lte;

                                        DEBUG("Stream of length %"PRIu64" is duplicate "
                                              "with one already in WIM",
                                        DEBUG("Stream of length %"PRIu64" is duplicate "
                                              "with one already in WIM",

-                                             wim_resource_size(duplicate_lte));
+                                             wim_resource_size(lte));

                                } else {
                                        DEBUG("Discarding duplicate stream of length %"PRIu64,
                                } else {
                                        DEBUG("Discarding duplicate stream of length %"PRIu64,

-                                             wim_resource_size(duplicate_lte));
+                                             wim_resource_size(lte));

                                        goto skip_to_progress;
                                }
 
                                        goto skip_to_progress;
                                }
 


@@ -1306,6 +1305,9 @@ write_stream_list(struct list_head *stream_list,
        union wimlib_progress_info progress;
        int ret;
 
        union wimlib_progress_info progress;
        int ret;
 

+       if (list_empty(stream_list))
+               return 0;
+

        list_for_each_entry(lte, stream_list, write_streams_list) {
                num_streams++;
                total_bytes += wim_resource_size(lte);
        list_for_each_entry(lte, stream_list, write_streams_list) {
                num_streams++;
                total_bytes += wim_resource_size(lte);


@@ -1346,58 +1348,6 @@ write_stream_list(struct list_head *stream_list,
        return ret;
 }
 
        return ret;
 }
 

-struct lte_overwrite_prepare_args {
-       WIMStruct *wim;
-       off_t end_offset;
-       struct list_head *stream_list;
-};
-
-static int
-lte_overwrite_prepare(struct wim_lookup_table_entry *lte, void *arg)
-{
-       struct lte_overwrite_prepare_args *args = arg;
-
-       if (lte->resource_location == RESOURCE_IN_WIM &&
-           lte->wim == args->wim &&
-           lte->resource_entry.offset + lte->resource_entry.size > args->end_offset)
-       {
-       #ifdef ENABLE_ERROR_MESSAGES
-               ERROR("The following resource is after the XML data:");
-               print_lookup_table_entry(lte, stderr);
-       #endif
-               return WIMLIB_ERR_RESOURCE_ORDER;
-       }
-
-       lte->out_refcnt = lte->refcnt;
-       memcpy(&lte->output_resource_entry, &lte->resource_entry,
-              sizeof(struct resource_entry));
-       if (!(lte->resource_entry.flags & WIM_RESHDR_FLAG_METADATA))
-               if (lte->resource_location != RESOURCE_IN_WIM || lte->wim != args->wim)
-                       list_add(&lte->staging_list, args->stream_list);
-       return 0;
-}
-
-static int
-wim_prepare_streams(WIMStruct *wim, off_t end_offset,
-                   struct list_head *stream_list)
-{
-       struct lte_overwrite_prepare_args args = {
-               .wim         = wim,
-               .end_offset  = end_offset,
-               .stream_list = stream_list,
-       };
-       int ret;
-
-       for (int i = 0; i < wim->hdr.image_count; i++) {
-               ret = lte_overwrite_prepare(wim->image_metadata[i]->metadata_lte,
-                                           &args);
-               if (ret)
-                       return ret;
-       }
-       return for_lookup_table_entry(wim->lookup_table,
-                                     lte_overwrite_prepare, &args);
-}
-

 struct stream_size_table {
        struct hlist_head *array;
        size_t num_entries;
 struct stream_size_table {
        struct hlist_head *array;
        size_t num_entries;


@@ -1445,6 +1395,111 @@ stream_size_table_insert(struct wim_lookup_table_entry *lte, void *_tab)
 }
 
 
 }
 
 

+struct lte_overwrite_prepare_args {
+       WIMStruct *wim;
+       off_t end_offset;
+       struct list_head *stream_list;
+       struct stream_size_table *stream_size_tab;
+};
+
+static int
+lte_overwrite_prepare(struct wim_lookup_table_entry *lte, void *arg)
+{
+       struct lte_overwrite_prepare_args *args = arg;
+
+       if (lte->resource_location == RESOURCE_IN_WIM &&
+           lte->wim == args->wim)
+       {
+               /* We can't do an in place overwrite on the WIM if there are
+                * streams after the XML data. */
+               if (lte->resource_entry.offset +
+                   lte->resource_entry.size > args->end_offset)
+               {
+               #ifdef ENABLE_ERROR_MESSAGES
+                       ERROR("The following resource is after the XML data:");
+                       print_lookup_table_entry(lte, stderr);
+               #endif
+                       return WIMLIB_ERR_RESOURCE_ORDER;
+               }
+       } else {
+               if (!(lte->resource_entry.flags & WIM_RESHDR_FLAG_METADATA))
+                       list_add_tail(&lte->write_streams_list, args->stream_list);
+       }
+       lte->out_refcnt = lte->refcnt;
+       stream_size_table_insert(lte, args->stream_size_tab);
+       return 0;
+}
+
+static int
+lte_set_output_res_entry(struct wim_lookup_table_entry *lte, void *_wim)
+{
+       if (lte->resource_location == RESOURCE_IN_WIM &&
+           lte->wim == _wim)
+       {
+               memcpy(&lte->output_resource_entry, &lte->resource_entry,
+                      sizeof(struct resource_entry));
+       }
+       return 0;
+}
+
+/* Given a WIM that we are going to overwrite in place with zero or more
+ * additional streams added, construct a list the list of new unique streams
+ * ('struct wim_lookup_table_entry's) that must be written, plus any unhashed
+ * streams that need to be added but may be identical to other hashed or
+ * unhashed streams.  These unhashed streams are checksummed while the streams
+ * are being written.  To aid this process, the member @unique_size is set to 1
+ * on streams that have a unique size and therefore must be written.
+ *
+ * The out_refcnt member of each 'struct wim_lookup_table_entry' is set to
+ * indicate the number of times the stream is referenced in only the streams
+ * that are being written; this may still be adjusted later when unhashed
+ * streams are being resolved.
+ */
+static int
+prepare_streams_for_overwrite(WIMStruct *wim, off_t end_offset,
+                             struct list_head *stream_list)
+{
+       int ret;
+       struct stream_size_table stream_size_tab;
+       struct lte_overwrite_prepare_args args = {
+               .wim         = wim,
+               .end_offset  = end_offset,
+               .stream_list = stream_list,
+               .stream_size_tab = &stream_size_tab,
+       };
+
+       ret = init_stream_size_table(&stream_size_tab, 9001);
+       if (ret)
+               return ret;
+
+       INIT_LIST_HEAD(stream_list);
+       for (int i = 0; i < wim->hdr.image_count; i++) {
+               struct wim_image_metadata *imd;
+               struct wim_lookup_table_entry *lte;
+
+               imd = wim->image_metadata[i];
+               image_for_each_unhashed_stream(lte, imd) {
+                       ret = lte_overwrite_prepare(lte, &args);
+                       if (ret)
+                               goto out_destroy_stream_size_table;
+               }
+       }
+       ret = for_lookup_table_entry(wim->lookup_table,
+                                    lte_overwrite_prepare, &args);
+       if (ret)
+               goto out_destroy_stream_size_table;
+
+       for (int i = 0; i < wim->hdr.image_count; i++)
+               lte_set_output_res_entry(wim->image_metadata[i]->metadata_lte,
+                                        wim);
+       ret = for_lookup_table_entry(wim->lookup_table,
+                                    lte_set_output_res_entry, wim);
+out_destroy_stream_size_table:
+       destroy_stream_size_table(&stream_size_tab);
+       return ret;
+}
+
+

 struct find_streams_ctx {
        struct list_head stream_list;
        struct stream_size_table stream_size_tab;
 struct find_streams_ctx {
        struct list_head stream_list;
        struct stream_size_table stream_size_tab;


@@ -1946,18 +2001,17 @@ overwrite_wim_inplace(WIMStruct *w, int write_flags,
                write_flags |= WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE |
                               WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML;
        }
                write_flags |= WIMLIB_WRITE_FLAG_NO_LOOKUP_TABLE |
                               WIMLIB_WRITE_FLAG_CHECKPOINT_AFTER_XML;
        }

-       INIT_LIST_HEAD(&stream_list);
-       ret = wim_prepare_streams(w, old_wim_end, &stream_list);
-       if (ret != 0)
+       ret = prepare_streams_for_overwrite(w, old_wim_end, &stream_list);
+       if (ret)

                return ret;
 
        ret = open_wim_writable(w, w->filename, false,
                                (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) != 0);
                return ret;
 
        ret = open_wim_writable(w, w->filename, false,
                                (write_flags & WIMLIB_WRITE_FLAG_CHECK_INTEGRITY) != 0);

-       if (ret != 0)
+       if (ret)

                return ret;
 
        ret = lock_wim(w, w->out_fp);
                return ret;
 
        ret = lock_wim(w, w->out_fp);

-       if (ret != 0) {
+       if (ret) {

                fclose(w->out_fp);
                w->out_fp = NULL;
                return ret;
                fclose(w->out_fp);
                w->out_fp = NULL;
                return ret;


@@ -1971,20 +2025,16 @@ overwrite_wim_inplace(WIMStruct *w, int write_flags,
                return WIMLIB_ERR_WRITE;
        }
 
                return WIMLIB_ERR_WRITE;
        }
 

-       if (!list_empty(&stream_list)) {
-               DEBUG("Writing newly added streams (offset = %"PRIu64")",
-                     old_wim_end);
-               ret = write_stream_list(&stream_list,
-                                       w->lookup_table,
-                                       w->out_fp,
-                                       wimlib_get_compression_type(w),
-                                       write_flags, num_threads,
-                                       progress_func);
-               if (ret != 0)
-                       goto out_ftruncate;
-       } else {
-               DEBUG("No new streams were added");
-       }
+       DEBUG("Writing newly added streams (offset = %"PRIu64")",
+             old_wim_end);
+       ret = write_stream_list(&stream_list,
+                               w->lookup_table,
+                               w->out_fp,
+                               wimlib_get_compression_type(w),
+                               write_flags, num_threads,
+                               progress_func);
+       if (ret)
+               goto out_ftruncate;

 
        for (int i = 0; i < w->hdr.image_count; i++) {
                if (w->image_metadata[i]->modified) {
 
        for (int i = 0; i < w->hdr.image_count; i++) {
                if (w->image_metadata[i]->modified) {

diff --git a/src/xpress-compress.c b/src/xpress-compress.c
index 68d2890656040be259f57f5b5bc69beaf354b52b..21118e7652b2e17d2287c085cc7e571c4bf88c3c 100644 (file)
--- a/src/xpress-compress.c
+++ b/src/xpress-compress.c
@@ -146,7 +146,6 @@ WIMLIBAPI unsigned
 wimlib_xpress_compress(const void *__uncompressed_data,
                       unsigned uncompressed_len, void *__compressed_data)
 {
 wimlib_xpress_compress(const void *__uncompressed_data,
                       unsigned uncompressed_len, void *__compressed_data)
 {

-       const u8 *uncompressed_data = __uncompressed_data;

        u8 *compressed_data = __compressed_data;
        struct output_bitstream ostream;
        u32 match_tab[uncompressed_len];
        u8 *compressed_data = __compressed_data;
        struct output_bitstream ostream;
        u32 match_tab[uncompressed_len];


@@ -157,6 +156,10 @@ wimlib_xpress_compress(const void *__uncompressed_data,
        unsigned compressed_len;
        unsigned i;
        int ret;
        unsigned compressed_len;
        unsigned i;
        int ret;

+       u8 uncompressed_data[uncompressed_len + 8];
+
+       memcpy(uncompressed_data, __uncompressed_data, uncompressed_len);
+       memset(uncompressed_data + uncompressed_len, 0, 8);

 
        wimlib_assert(uncompressed_len <= 32768);
 
 
        wimlib_assert(uncompressed_len <= 32768);