X-Git-Url: https://wimlib.net/git/?a=blobdiff_plain;f=src%2Fwrite.c;h=f397cbb280df33b2e1fb515b6f35094cd8662a1e;hb=0c2a83873e2c1e9ecea018227b14677aa29dc574;hp=b8d86e85fc7bf6540630ac8f0f1d950b64c226c9;hpb=b6034a5dd44709341c46d553b1c0294ec91f13e4;p=wimlib

diff --git a/src/write.c b/src/write.c
index b8d86e85..f397cbb2 100644
--- a/src/write.c
+++ b/src/write.c
@@ -39,6 +39,7 @@
 #include "wimlib/error.h"
 #include "wimlib/file_io.h"
 #include "wimlib/header.h"
+#include "wimlib/inode.h"
 #include "wimlib/integrity.h"
 #include "wimlib/lookup_table.h"
 #include "wimlib/metadata.h"
@@ -168,7 +169,6 @@ can_raw_copy(const struct wim_lookup_table_entry *lte,
 	if (rspec->is_pipable != !!(write_resource_flags & WRITE_RESOURCE_FLAG_PIPABLE))
 		return false;
 
-
 	if (rspec->flags & WIM_RESHDR_FLAG_COMPRESSED) {
 		/* Normal compressed resource: Must use same compression type
 		 * and chunk size.  */
@@ -176,15 +176,12 @@ can_raw_copy(const struct wim_lookup_table_entry *lte,
 			rspec->chunk_size == out_chunk_size);
 	}
 
-	/* XXX: For compatibility, we can't allow multiple packed resources per
-	 * WIM.  */
-#if 0
 	if ((rspec->flags & WIM_RESHDR_FLAG_PACKED_STREAMS) &&
 	    (write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS))
 	{
 		/* Packed resource: Such resources may contain multiple streams,
 		 * and in general only a subset of them need to be written.  As
-		 * a heuristic, re-use the raw data if at least half the
+		 * a heuristic, re-use the raw data if more than two-thirds the
 		 * uncompressed size is being written.  */
 
 		/* Note: packed resources contain a header that specifies the
@@ -199,9 +196,8 @@ can_raw_copy(const struct wim_lookup_table_entry *lte,
 			if (res_stream->will_be_in_output_wim)
 				write_size += res_stream->size;
 
-		return (write_size > rspec->uncompressed_size / 2);
+		return (write_size > rspec->uncompressed_size * 2 / 3);
 	}
-#endif
 
 	return false;
 }
@@ -233,7 +229,7 @@ stream_set_out_reshdr_for_reuse(struct wim_lookup_table_entry *lte)
 
 		lte->out_res_offset_in_wim = rspec->offset_in_wim;
 		lte->out_res_size_in_wim = rspec->size_in_wim;
-		/*lte->out_res_uncompressed_size = rspec->uncompressed_size;*/
+		lte->out_res_uncompressed_size = rspec->uncompressed_size;
 	} else {
 		wimlib_assert(!(lte->flags & WIM_RESHDR_FLAG_PACKED_STREAMS));
 
@@ -277,7 +273,6 @@ struct write_streams_progress_data {
 	wimlib_progress_func_t progress_func;
 	union wimlib_progress_info progress;
 	uint64_t next_progress;
-	WIMStruct *prev_wim_part;
 };
 
 static void
@@ -288,7 +283,6 @@ do_write_streams_progress(struct write_streams_progress_data *progress_data,
 			  bool discarded)
 {
 	union wimlib_progress_info *progress = &progress_data->progress;
-	bool new_wim_part;
 
 	if (discarded) {
 		progress->write_streams.total_bytes -= complete_size;
@@ -303,20 +297,8 @@ do_write_streams_progress(struct write_streams_progress_data *progress_data,
 		progress->write_streams.completed_streams += complete_count;
 	}
 
-	new_wim_part = false;
-	if (cur_stream->resource_location == RESOURCE_IN_WIM &&
-	    cur_stream->rspec->wim != progress_data->prev_wim_part)
-	{
-		if (progress_data->prev_wim_part) {
-			new_wim_part = true;
-			progress->write_streams.completed_parts++;
-		}
-		progress_data->prev_wim_part = cur_stream->rspec->wim;
-	}
-
 	if (progress_data->progress_func
-	    && (progress->write_streams.completed_bytes >= progress_data->next_progress
-		|| new_wim_part))
+	    && (progress->write_streams.completed_bytes >= progress_data->next_progress))
 	{
 		progress_data->progress_func(WIMLIB_PROGRESS_MSG_WRITE_STREAMS,
 					     progress);
@@ -568,8 +550,8 @@ end_chunk_table(struct write_streams_ctx *ctx, u64 res_actual_size,
 			hdr.chunk_size = cpu_to_le32(ctx->out_chunk_size);
 			hdr.compression_format = cpu_to_le32(ctx->out_ctype);
 
-			BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 1);
-			BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 2);
+			BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_XPRESS != 1);
+			BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZX != 2);
 			BUILD_BUG_ON(WIMLIB_COMPRESSION_TYPE_LZMS != 3);
 
 			ret = full_pwrite(ctx->out_fd, &hdr, sizeof(hdr),
@@ -631,7 +613,7 @@ end_write_resource(struct write_streams_ctx *ctx, struct wim_reshdr *out_reshdr)
 /* Begin processing a stream for writing.  */
 static int
 write_stream_begin_read(struct wim_lookup_table_entry *lte,
-			bool is_partial_res, void *_ctx)
+			u32 flags, void *_ctx)
 {
 	struct write_streams_ctx *ctx = _ctx;
 	int ret;
@@ -655,7 +637,7 @@ write_stream_begin_read(struct wim_lookup_table_entry *lte,
 	ctx->stream_was_duplicate = false;
 	if (ctx->lookup_table != NULL && lte->unhashed && !lte->unique_size) {
 
-		wimlib_assert(!is_partial_res);
+		wimlib_assert(!(flags & BEGIN_STREAM_FLAG_PARTIAL_RESOURCE));
 
 		struct wim_lookup_table_entry *lte_new;
 
@@ -914,8 +896,8 @@ submit_chunk_for_compression(struct write_streams_ctx *ctx,
 	 * compressed chunk.  */
 	while (!ctx->compressor->submit_chunk(ctx->compressor, chunk, size)) {
 		const void *cchunk;
-		unsigned csize;
-		unsigned usize;
+		u32 csize;
+		u32 usize;
 		bool bret;
 		int ret;
 
@@ -1060,24 +1042,23 @@ compute_stream_list_stats(struct list_head *stream_list,
 	ctx->progress_data.progress.write_streams.total_parts       = total_parts;
 	ctx->progress_data.progress.write_streams.completed_parts   = 0;
 	ctx->progress_data.next_progress = 0;
-	ctx->progress_data.prev_wim_part = NULL;
 }
 
 /* Find streams in @stream_list that can be copied to the output WIM in raw form
- * rather than compressed.  Delete these streams from @stream_list, and move one
- * per resource to @raw_copy_resources.  Return the total uncompressed size of
- * the streams that need to be compressed.  */
+ * rather than compressed.  Delete these streams from @stream_list and move them
+ * to @raw_copy_streams.  Return the total uncompressed size of the streams that
+ * need to be compressed.  */
 static u64
-find_raw_copy_resources(struct list_head *stream_list,
-			int write_resource_flags,
-			int out_ctype,
-			u32 out_chunk_size,
-			struct list_head *raw_copy_resources)
+find_raw_copy_streams(struct list_head *stream_list,
+		      int write_resource_flags,
+		      int out_ctype,
+		      u32 out_chunk_size,
+		      struct list_head *raw_copy_streams)
 {
 	struct wim_lookup_table_entry *lte, *tmp;
 	u64 num_bytes_to_compress = 0;
 
-	INIT_LIST_HEAD(raw_copy_resources);
+	INIT_LIST_HEAD(raw_copy_streams);
 
 	/* Initialize temporary raw_copy_ok flag.  */
 	list_for_each_entry(lte, stream_list, write_streams_list)
@@ -1088,13 +1069,14 @@ find_raw_copy_resources(struct list_head *stream_list,
 		if (lte->resource_location == RESOURCE_IN_WIM &&
 		    lte->rspec->raw_copy_ok)
 		{
-			list_del(&lte->write_streams_list);
+			list_move_tail(&lte->write_streams_list,
+				       raw_copy_streams);
 		} else if (can_raw_copy(lte, write_resource_flags,
 				 out_ctype, out_chunk_size))
 		{
 			lte->rspec->raw_copy_ok = 1;
 			list_move_tail(&lte->write_streams_list,
-				       raw_copy_resources);
+				       raw_copy_streams);
 		} else {
 			num_bytes_to_compress += lte->size;
 		}
@@ -1165,20 +1147,28 @@ write_raw_copy_resource(struct wim_resource_spec *in_rspec,
 	return 0;
 }
 
-/* Copy a list of raw compressed resources located other WIM file(s) to the WIM
- * file being written.  */
+/* Copy a list of raw compressed resources located in other WIM file(s) to the
+ * WIM file being written.  */
 static int
-write_raw_copy_resources(struct list_head *raw_copy_resources,
+write_raw_copy_resources(struct list_head *raw_copy_streams,
 			 struct filedes *out_fd,
 			 struct write_streams_progress_data *progress_data)
 {
 	struct wim_lookup_table_entry *lte;
 	int ret;
 
-	list_for_each_entry(lte, raw_copy_resources, write_streams_list) {
-		ret = write_raw_copy_resource(lte->rspec, out_fd);
-		if (ret)
-			return ret;
+	list_for_each_entry(lte, raw_copy_streams, write_streams_list)
+		lte->rspec->raw_copy_ok = 1;
+
+	list_for_each_entry(lte, raw_copy_streams, write_streams_list) {
+		if (lte->rspec->raw_copy_ok) {
+			/* Write each packed resource only one time, no matter
+			 * how many streams reference it.  */
+			ret = write_raw_copy_resource(lte->rspec, out_fd);
+			if (ret)
+				return ret;
+			lte->rspec->raw_copy_ok = 0;
+		}
 		do_write_streams_progress(progress_data, lte, lte->size,
 					  1, false);
 	}
@@ -1190,8 +1180,8 @@ static int
 finish_remaining_chunks(struct write_streams_ctx *ctx)
 {
 	const void *cdata;
-	unsigned csize;
-	unsigned usize;
+	u32 csize;
+	u32 usize;
 	int ret;
 
 	if (ctx->compressor == NULL)
@@ -1355,7 +1345,7 @@ write_stream_list(struct list_head *stream_list,
 {
 	int ret;
 	struct write_streams_ctx ctx;
-	struct list_head raw_copy_resources;
+	struct list_head raw_copy_streams;
 
 	wimlib_assert((write_resource_flags &
 		       (WRITE_RESOURCE_FLAG_PACK_STREAMS |
@@ -1405,11 +1395,11 @@ write_stream_list(struct list_head *stream_list,
 
 	ctx.progress_data.progress_func = progress_func;
 
-	ctx.num_bytes_to_compress = find_raw_copy_resources(stream_list,
-							    write_resource_flags,
-							    out_ctype,
-							    out_chunk_size,
-							    &raw_copy_resources);
+	ctx.num_bytes_to_compress = find_raw_copy_streams(stream_list,
+							  write_resource_flags,
+							  out_ctype,
+							  out_chunk_size,
+							  &raw_copy_streams);
 
 	DEBUG("Writing stream list "
 	      "(offset = %"PRIu64", write_resource_flags=0x%08x, "
@@ -1429,11 +1419,11 @@ write_stream_list(struct list_head *stream_list,
 	 * to do compression.  There are serial and parallel implementations of
 	 * the chunk_compressor interface.  We default to parallel using the
 	 * specified number of threads, unless the upper bound on the number
-	 * bytes needing to be compressed is less 2000000 (heuristic value).  */
+	 * bytes needing to be compressed is less than a heuristic value.  */
 	if (out_ctype != WIMLIB_COMPRESSION_TYPE_NONE) {
 
 	#ifdef ENABLE_MULTITHREADED_COMPRESSION
-		if (ctx.num_bytes_to_compress >= 2000000) {
+		if (ctx.num_bytes_to_compress > max(2000000, out_chunk_size)) {
 			ret = new_parallel_chunk_compressor(out_ctype,
 							    out_chunk_size,
 							    num_threads, 0,
@@ -1524,7 +1514,7 @@ write_stream_list(struct list_head *stream_list,
 			lte->out_reshdr.offset_in_wim = offset_in_res;
 			lte->out_res_offset_in_wim = reshdr.offset_in_wim;
 			lte->out_res_size_in_wim = reshdr.size_in_wim;
-			/*lte->out_res_uncompressed_size = reshdr.uncompressed_size;*/
+			lte->out_res_uncompressed_size = reshdr.uncompressed_size;
 			offset_in_res += lte->size;
 		}
 		wimlib_assert(offset_in_res == reshdr.uncompressed_size);
@@ -1533,7 +1523,7 @@ write_stream_list(struct list_head *stream_list,
 out_write_raw_copy_resources:
 	/* Copy any compressed resources for which the raw data can be reused
 	 * without decompression.  */
-	ret = write_raw_copy_resources(&raw_copy_resources, ctx.out_fd,
+	ret = write_raw_copy_resources(&raw_copy_streams, ctx.out_fd,
 				       &ctx.progress_data);
 
 out_destroy_context:
@@ -1560,6 +1550,11 @@ wim_write_stream_list(WIMStruct *wim,
 
 	write_resource_flags = write_flags_to_resource_flags(write_flags);
 
+	/* wimlib v1.6.3: pack streams by default if the WIM version number is
+	 * that usually used in solid archives.  */
+	if (wim->hdr.wim_version == WIM_VERSION_PACKED_STREAMS)
+		write_resource_flags |= WRITE_RESOURCE_FLAG_PACK_STREAMS;
+
 	if (write_resource_flags & WRITE_RESOURCE_FLAG_PACK_STREAMS) {
 		out_chunk_size = wim->out_pack_chunk_size;
 		out_ctype = wim->out_pack_compression_type;
@@ -1842,7 +1837,7 @@ determine_stream_size_uniquity(struct list_head *stream_list,
 	struct stream_size_table tab;
 	struct wim_lookup_table_entry *lte;
 
-	ret = init_stream_size_table(&tab, lt->capacity);
+	ret = init_stream_size_table(&tab, 9001);
 	if (ret)
 		return ret;
 
@@ -2818,10 +2813,11 @@ wimlib_write(WIMStruct *wim, const tchar *path,
 	     int image, int write_flags, unsigned num_threads,
 	     wimlib_progress_func_t progress_func)
 {
-	if (!path)
+	if (write_flags & ~WIMLIB_WRITE_MASK_PUBLIC)
 		return WIMLIB_ERR_INVALID_PARAM;
 
-	write_flags &= WIMLIB_WRITE_MASK_PUBLIC;
+	if (path == NULL || path[0] == T('\0'))
+		return WIMLIB_ERR_INVALID_PARAM;
 
 	return write_standalone_wim(wim, path, image, write_flags,
 				    num_threads, progress_func);
@@ -2833,10 +2829,12 @@ wimlib_write_to_fd(WIMStruct *wim, int fd,
 		   int image, int write_flags, unsigned num_threads,
 		   wimlib_progress_func_t progress_func)
 {
+	if (write_flags & ~WIMLIB_WRITE_MASK_PUBLIC)
+		return WIMLIB_ERR_INVALID_PARAM;
+
 	if (fd < 0)
 		return WIMLIB_ERR_INVALID_PARAM;
 
-	write_flags &= WIMLIB_WRITE_MASK_PUBLIC;
 	write_flags |= WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR;
 
 	return write_standalone_wim(wim, &fd, image, write_flags,
@@ -3122,7 +3120,10 @@ overwrite_wim_via_tmpfile(WIMStruct *wim, int write_flags,
 		return ret;
 	}
 
-	close_wim(wim);
+	if (filedes_valid(&wim->in_fd)) {
+		filedes_close(&wim->in_fd);
+		filedes_invalidate(&wim->in_fd);
+	}
 
 	/* Rename the new WIM file to the original WIM file.  Note: on Windows
 	 * this actually calls win32_rename_replacement(), not _wrename(), so
@@ -3168,14 +3169,6 @@ can_overwrite_wim_inplace(const WIMStruct *wim, int write_flags)
 	if (wim_is_pipable(wim) || (write_flags & WIMLIB_WRITE_FLAG_PIPABLE))
 		return false;
 
-	/* wimlib allows multiple packs in a single WIM, but they don't seem to
-	 * be compatible with WIMGAPI, so force all streams to be repacked if
-	 * the WIM already may have contained a pack and PACK_STREAMS was
-	 * requested.  */
-	if (write_flags & WIMLIB_WRITE_FLAG_PACK_STREAMS &&
-	    wim->hdr.wim_version == WIM_VERSION_PACKED_STREAMS)
-		return false;
-
 	/* The default compression type and compression chunk size selected for
 	 * the output WIM must be the same as those currently used for the WIM.
 	 */
@@ -3196,9 +3189,7 @@ wimlib_overwrite(WIMStruct *wim, int write_flags,
 	int ret;
 	u32 orig_hdr_flags;
 
-	write_flags &= WIMLIB_WRITE_MASK_PUBLIC;
-
-	if (write_flags & WIMLIB_WRITE_FLAG_FILE_DESCRIPTOR)
+	if (write_flags & ~WIMLIB_WRITE_MASK_PUBLIC)
 		return WIMLIB_ERR_INVALID_PARAM;
 
 	if (!wim->filename)