Context Navigation

← Previous Revision
Latest Revision
Next Revision →
Blame
Revision Log
Repository URL

xdelta3-main.h @ 185

Last change on this file since 185 was 185, checked in by geyser, 14 years ago

File size: 87.3 KB

Line
1	/* xdelta 3 - delta compression tools and library
2	* Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007,
3	* Joshua P. MacDonald
4	*
5	* This program is free software; you can redistribute it and/or modify
6	* it under the terms of the GNU General Public License as published by
7	* the Free Software Foundation; either version 2 of the License, or
8	* (at your option) any later version.
9	*
10	* This program is distributed in the hope that it will be useful,
11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13	* GNU General Public License for more details.
14	*
15	* You should have received a copy of the GNU General Public License
16	* along with this program; if not, write to the Free Software
17	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18	*/
19
20	/* This is all the extra stuff you need for convenience to users in a command line
21	* application. It contains these major components:
22	*
23	* 1. VCDIFF tools
24	* 2. external compression support (this is POSIX-specific).
25	* 3. a general read/write loop that handles all of the Xdelta decode/encode/VCDIFF-print
26	* functions
27	* 4. command-line interpreter
28	* 5. an Xdelta application header which stores default filename, external compression settings
29	* 6. output/error printing
30	* 7. basic file support and OS interface
31	*/
32
33	/* TODO list:
34	* 1. do exact gzip-like filename, stdout handling. make a .vcdiff extension, refuse
35	* to encode to stdout without -cf, etc.
36	* 2. Allow the user to add a comment string to the app header without disturbing the default
37	* behavior.
38	* 3. "Source file must be seekable" is not actually true for encoding, given current
39	* behavior. Allow non-seekable sources? It would in theory let you use a fifo for
40	* the source.
41	*/
42
43	/* On error handling and printing:
44	*
45	* The xdelta library sets stream->msg to indicate what condition caused an internal
46	* failure, but many failures originate here and are printed here. The return convention
47	* is 0 for success, as throughout Xdelta code, but special attention is required here for
48	* the operating system calls with different error handling. See the main_file_* routines.
49	* All errors in this file have a message printed at the time of occurance. Since some of
50	* these calls occur within calls to the library, the error may end up being printed again
51	* with a more general error message.
52	*/
53
54	/******************************************************************************************/
55
56	#ifndef XD3_POSIX
57	#define XD3_POSIX 0
58	#endif
59	#ifndef XD3_STDIO
60	#define XD3_STDIO 0
61	#endif
62	#ifndef XD3_WIN32
63	#define XD3_WIN32 0
64	#endif
65
66	/* Combines xd3_strerror() and strerror() */
67	const char* xd3_mainerror(int err_num);
68
69	/* XPRINTX (used by main) prefixes an "xdelta3: " to the output. */
70	#define XPR fprintf
71	#define NT stderr, "xdelta3: "
72
73	/* If none are set, default to posix. */
74	#if (XD3_POSIX + XD3_STDIO + XD3_WIN32) == 0
75	#undef XD3_POSIX
76	#define XD3_POSIX 1
77	#endif
78
79	/* Handle externally-compressed inputs. */
80	#ifndef EXTERNAL_COMPRESSION
81	#define EXTERNAL_COMPRESSION 1
82	#endif
83
84	#define PRINTHDR_SPECIAL -4378291
85
86	/* The number of soft-config variables. */
87	#define XD3_SOFTCFG_VARCNT 7
88
89	/* this is used as in XPR(NT XD3_LIB_ERRMSG (stream, ret)) to print an error message
90	* from the library. */
91	#define XD3_LIB_ERRMSG(stream, ret) "%s: %s\n", xd3_errstring (stream), xd3_mainerror (ret)
92
93	#include <stdio.h> /* fprintf */
94
95	#if XD3_POSIX
96	#include <unistd.h> /* close, read, write... */
97	#include <sys/types.h>
98	#include <fcntl.h>
99	#endif
100
101	#ifndef _WIN32
102	#include <unistd.h> /* lots */
103	#include <sys/time.h> /* gettimeofday() */
104	#include <sys/stat.h> /* stat() and fstat() */
105	#else
106	#define strtoll _strtoi64
107	#include <sys/types.h>
108	#include <sys/stat.h>
109	#ifndef WIFEXITED
110	# define WIFEXITED(stat) (((((int ) &(stat))) & 0xff) == 0)
111	#endif
112	#ifndef WEXITSTATUS
113	# define WEXITSTATUS(stat) (((((int ) &(stat))) >> 8) & 0xff)
114	#endif
115	#ifndef S_ISREG
116	//# ifdef S_IFREG
117	//# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
118	//# else
119	# define S_ISREG(m) 1
120	//# endif
121	#endif /* !S_ISREG */
122
123	// For standard input/output handles
124	static STARTUPINFO winStartupInfo;
125	#endif
126
127	/******************************************************************************************
128	ENUMS and TYPES
129	******************************************************************************************/
130
131	/* These flags (mainly pertaining to main_read() operations) are set in the
132	* main_file->flags variable. All are related to with external decompression support.
133	*
134	* RD_FIRST causes the external decompression check when the input is first read.
135	*
136	* RD_NONEXTERNAL disables external decompression for reading a compressed input, in the
137	* case of Xdelta inputs. Note: Xdelta is supported as an external compression type,
138	* which makes is the reason for this flag. An example to justify this is: to create a
139	* delta between two files that are VCDIFF-compressed. Two external Xdelta decoders are
140	* run to supply decompressed source and target inputs to the Xdelta encoder. */
141	typedef enum
142	{
143	RD_FIRST = (1 << 0),
144	RD_NONEXTERNAL = (1 << 1),
145	RD_EXTERNAL_V1 = (1 << 2),
146	} xd3_read_flags;
147
148	/* main_file->mode values */
149	typedef enum
150	{
151	XO_READ = 0,
152	XO_WRITE = 1,
153	} main_file_modes;
154
155	/* Main commands. For example, CMD_PRINTHDR is the "xdelta printhdr" command. */
156	typedef enum
157	{
158	CMD_NONE = 0,
159	CMD_PRINTHDR,
160	CMD_PRINTHDRS,
161	CMD_PRINTDELTA,
162	#if XD3_ENCODER
163	CMD_ENCODE,
164	#endif
165	CMD_DECODE,
166	CMD_TEST,
167	CMD_CONFIG,
168	} xd3_cmd;
169
170	#if XD3_ENCODER
171	#define CMD_DEFAULT CMD_ENCODE
172	#define IS_ENCODE(cmd) (cmd == CMD_ENCODE)
173	#else
174	#define CMD_DEFAULT CMD_DECODE
175	#define IS_ENCODE(cmd) (0)
176	#endif
177
178	typedef struct _main_file main_file;
179	typedef struct _main_extcomp main_extcomp;
180	typedef struct _main_blklru main_blklru;
181	typedef struct _main_blklru_list main_blklru_list;
182
183	/* The main_file object supports abstract system calls like open, close, read, write, seek,
184	* stat. The program uses these to represent both seekable files and non-seekable files.
185	* Source files must be seekable, but the target input and any output file do not require
186	* seekability.
187	*/
188	struct _main_file
189	{
190	#if XD3_STDIO
191	FILE *file;
192	#elif XD3_POSIX
193	int file;
194	#elif XD3_WIN32
195	HANDLE file;
196	#endif
197
198	int mode; /* XO_READ and XO_WRITE */
199	const char filename; / File name or /dev/stdin, /dev/stdout, /dev/stderr. */
200	char filename_copy; / File name or /dev/stdin, /dev/stdout, /dev/stderr. */
201	const char realname; / File name or /dev/stdin, /dev/stdout, /dev/stderr. */
202	const main_extcomp compressor; / External compression struct. */
203	int flags; /* RD_FIRST, RD_NONEXTERNAL, ... */
204	xoff_t nread; /* for input position */
205	xoff_t nwrite; /* for output position */
206	uint8_t snprintf_buf; / internal snprintf() use */
207	};
208
209	/* Various strings and magic values used to detect and call external compression. See
210	* below for examples. */
211	struct _main_extcomp
212	{
213	const char *recomp_cmdname;
214	const char *recomp_options;
215
216	const char *decomp_cmdname;
217	const char *decomp_options;
218
219	const char *ident;
220	const char *magic;
221	int magic_size;
222	int flags;
223	};
224
225	/* This file implements a small LRU of source blocks. For encoding purposes,
226	* we prevent paging in blocks we've already scanned in the source (return
227	* XD3_NOTAVAIL). */
228	struct _main_blklru_list
229	{
230	main_blklru_list *next;
231	main_blklru_list *prev;
232	};
233
234	struct _main_blklru
235	{
236	uint8_t *blk;
237	xoff_t blkno;
238	main_blklru_list link;
239	};
240
241	#define LRU_SIZE 32U
242	#define XD3_MINSRCWINSZ XD3_ALLOCSIZE
243
244	/* ... represented as a list (no cache index). */
245	XD3_MAKELIST(main_blklru_list,main_blklru,link);
246
247	// TODO:
248	// struct _main_state
249	// {
250
251	/* Program options: various command line flags and options. */
252	static int option_stdout = 0;
253	static int option_force = 0;
254	static int option_verbose = 0;
255	static int option_quiet = 0;
256	static int option_use_appheader = 1;
257	static uint8_t* option_appheader = NULL;
258	static int option_use_secondary = 0;
259	static char* option_secondary = NULL;
260	static int option_use_checksum = 1;
261	static int option_use_altcodetable = 0;
262	static char* option_smatch_config = NULL;
263	static int option_no_compress = 0;
264	static int option_no_output = 0; /* do not open or write output */
265	static const char *option_source_filename = NULL;
266
267	static int option_level = XD3_DEFAULT_LEVEL;
268	static usize_t option_iopt_size = XD3_DEFAULT_IOPT_SIZE;
269	static usize_t option_winsize = XD3_DEFAULT_WINSIZE;
270	static usize_t option_srcwinsz = XD3_DEFAULT_SRCWINSZ;
271	static usize_t option_sprevsz = XD3_DEFAULT_SPREVSZ;
272
273	/* These variables are supressed to avoid their use w/o support. main() warns
274	* appropriately. */
275	#if EXTERNAL_COMPRESSION
276	static int option_decompress_inputs = 1;
277	static int option_recompress_outputs = 1;
278	#endif
279
280	/* This is for comparing "printdelta" output without attention to
281	* copy-instruction modes. */
282	#if VCDIFF_TOOLS
283	static int option_print_cpymode = 1;
284	#endif
285
286	/* Static variables */
287	IF_DEBUG(static int main_mallocs = 0;)
288
289	static char* program_name = NULL;
290	static uint8_t* appheader_used = NULL;
291	static uint8_t* main_bdata = NULL;
292
293	/* The LRU: obviously this is shared by all callers. */
294	static int lru_size = 0;
295	static main_blklru lru = NULL; / array of lru_size elts */
296	static main_blklru_list lru_list;
297	static main_blklru_list lru_free;
298	static int do_not_lru = 0; /* set to avoid lru, instead discard oldest */
299
300	static int lru_hits = 0;
301	static int lru_misses = 0;
302	static int lru_filled = 0;
303
304	/* Hacks for VCDIFF tools */
305	static int allow_fake_source = 0;
306
307	/* This array of compressor types is compiled even if EXTERNAL_COMPRESSION is false just so
308	* the program knows the mapping of IDENT->NAME. */
309	static main_extcomp extcomp_types[] =
310	{
311	/* The entry for xdelta3 must be 0 because the program_name is set there. */
312	{ "xdelta3", "-cfq", "xdelta3", "-dcfq", "X", "\xd6\xc3\xc4", 3, RD_NONEXTERNAL },
313	{ "bzip2", "-cf", "bzip2", "-dcf", "B", "BZh", 3, 0 },
314	{ "gzip", "-cf", "gzip", "-dcf", "G", "\037\213", 2, 0 },
315	{ "compress", "-cf", "uncompress", "-cf", "Z", "\037\235", 2, 0 },
316
317	/* TODO: add commandline support for magic-less formats */
318	/{ "lzma", "-cf", "lzma", "-dcf", "M", "]\000", 2, 0 },/
319	};
320
321	// };
322
323	static void main_get_appheader (xd3_stream stream, main_file ifile,
324	main_file output, main_file sfile);
325
326	static int main_help (void);
327
328	static int
329	main_version (void)
330	{
331	/* $Format: " DP(RINT \"VERSION=3.$Xdelta3Version$\\n\");" $ */
332	DP(RINT "VERSION=3.0q\n");
333	return EXIT_SUCCESS;
334	}
335
336	static int
337	main_config (void)
338	{
339	main_version ();
340
341	DP(RINT "EXTERNAL_COMPRESSION=%d\n", EXTERNAL_COMPRESSION);
342	DP(RINT "GENERIC_ENCODE_TABLES=%d\n", GENERIC_ENCODE_TABLES);
343	DP(RINT "GENERIC_ENCODE_TABLES_COMPUTE=%d\n", GENERIC_ENCODE_TABLES_COMPUTE);
344	DP(RINT "REGRESSION_TEST=%d\n", REGRESSION_TEST);
345	DP(RINT "SECONDARY_DJW=%d\n", SECONDARY_DJW);
346	DP(RINT "SECONDARY_FGK=%d\n", SECONDARY_FGK);
347	DP(RINT "VCDIFF_TOOLS=%d\n", VCDIFF_TOOLS);
348	DP(RINT "XD3_ALLOCSIZE=%d\n", XD3_ALLOCSIZE);
349	DP(RINT "XD3_DEBUG=%d\n", XD3_DEBUG);
350	DP(RINT "XD3_ENCODER=%d\n", XD3_ENCODER);
351	DP(RINT "XD3_POSIX=%d\n", XD3_POSIX);
352	DP(RINT "XD3_STDIO=%d\n", XD3_STDIO);
353	DP(RINT "XD3_WIN32=%d\n", XD3_WIN32);
354	DP(RINT "XD3_USE_LARGEFILE64=%d\n", XD3_USE_LARGEFILE64);
355	DP(RINT "XD3_DEFAULT_LEVEL=%d\n", XD3_DEFAULT_LEVEL);
356	DP(RINT "XD3_DEFAULT_IOPT_SIZE=%d\n", XD3_DEFAULT_IOPT_SIZE);
357	DP(RINT "XD3_DEFAULT_SPREVSZ=%d\n", XD3_DEFAULT_SPREVSZ);
358	DP(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ);
359	DP(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE);
360	DP(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE);
361	DP(RINT "XD3_NODECOMPRESSSIZE=%d\n", XD3_NODECOMPRESSSIZE);
362
363	return EXIT_SUCCESS;
364	}
365
366	static void
367	reset_defaults(void)
368	{
369	option_stdout = 0;
370	option_force = 0;
371	option_verbose = 0;
372	option_quiet = 0;
373	option_appheader = NULL;
374	option_use_secondary = 0;
375	option_secondary = NULL;
376	option_use_altcodetable = 0;
377	option_smatch_config = NULL;
378	option_no_compress = 0;
379	option_no_output = 0;
380	option_source_filename = NULL;
381	program_name = NULL;
382	appheader_used = NULL;
383	main_bdata = NULL;
384	lru_size = 0;
385	lru = NULL;
386	do_not_lru = 0;
387	lru_hits = 0;
388	lru_misses = 0;
389	lru_filled = 0;
390	allow_fake_source = 0;
391	option_smatch_config = NULL;
392
393	option_use_appheader = 1;
394	option_use_checksum = 1;
395	#if EXTERNAL_COMPRESSION
396	option_decompress_inputs = 1;
397	option_recompress_outputs = 1;
398	#endif
399	#if VCDIFF_TOOLS
400	option_print_cpymode = 1;
401	#endif
402	option_level = XD3_DEFAULT_LEVEL;
403	option_iopt_size = XD3_DEFAULT_IOPT_SIZE;
404	option_winsize = XD3_DEFAULT_WINSIZE;
405	option_srcwinsz = XD3_DEFAULT_SRCWINSZ;
406	option_sprevsz = XD3_DEFAULT_SPREVSZ;
407	}
408
409	static void*
410	main_malloc1 (usize_t size)
411	{
412	void* r = malloc (size);
413	if (r == NULL) { XPR(NT "malloc: %s\n", xd3_mainerror (ENOMEM)); }
414	else if (option_verbose > 3) { XPR(NT "malloc: %u: %p\n", size, r); }
415	return r;
416	}
417
418	static void*
419	main_malloc (usize_t size)
420	{
421	void *r = main_malloc1 (size);
422	if (r) { IF_DEBUG (main_mallocs += 1); }
423	return r;
424	}
425
426	static void*
427	main_alloc (void *opaque,
428	usize_t items,
429	usize_t size)
430	{
431	return main_malloc1 (items * size);
432	}
433
434	static void
435	main_free1 (void opaque, void ptr)
436	{
437	if (option_verbose > 3) { XPR(NT "free: %p\n", ptr); }
438	free (ptr);
439	}
440
441	static void
442	main_free (void *ptr)
443	{
444	if (ptr)
445	{
446	IF_DEBUG (main_mallocs -= 1);
447	main_free1 (NULL, ptr);
448	IF_DEBUG (XD3_ASSERT(main_mallocs >= 0));
449	}
450	}
451
452	/* This ensures that (ret = errno) always indicates failure, in case errno was
453	* accidentally not set. If this prints there's a bug somewhere. */
454	static int
455	get_errno (void)
456	{
457	#ifndef _WIN32
458	if (errno == 0)
459	{
460	XPR(NT "you found a bug: expected errno != 0\n");
461	errno = XD3_INTERNAL;
462	}
463	return errno;
464	#else
465	DWORD errNum = GetLastError();
466	if (errNum == NO_ERROR) {
467	errNum = XD3_INTERNAL;
468	}
469	return errNum;
470	#endif
471	}
472
473	const char*
474	xd3_mainerror(int err_num) {
475	#ifndef _WIN32
476	const char* x = xd3_strerror (err_num);
477	if (x != NULL) {
478	return x;
479	}
480	return strerror(err_num);
481	#else
482	static char err_buf[256];
483	const char* x = xd3_strerror (err_num);
484	if (x != NULL) {
485	return x;
486	}
487	memset (err_buf, 0, 256);
488	FormatMessage (FORMAT_MESSAGE_FROM_SYSTEM \| FORMAT_MESSAGE_IGNORE_INSERTS,
489	NULL, err_num,
490	MAKELANGID (LANG_NEUTRAL, SUBLANG_DEFAULT),
491	err_buf, 256, NULL);
492	return err_buf;
493	#endif
494	}
495
496	static long
497	get_millisecs_now (void)
498	{
499	#ifndef _WIN32
500	struct timeval tv;
501
502	gettimeofday (& tv, NULL);
503
504	return (tv.tv_sec) * 1000L + (tv.tv_usec) / 1000;
505	#else
506	// Found this in an example on www.codeproject.com
507	// It doesn't matter that the offset is Jan 1, 1601
508	// Result is the numbre of 100 nanosecond units
509	// 100ns * 10,000 = 1ms
510	SYSTEMTIME st;
511	FILETIME ft;
512	__int64 pi = (__int64)&ft;
513	GetLocalTime(&st);
514	SystemTimeToFileTime(&st, &ft);
515	return (long)((*pi) / 10000);
516	#endif
517	}
518
519	/* Always >= 1 millisec, right? */
520	static long
521	get_millisecs_since (void)
522	{
523	static long last = 0;
524	long now = get_millisecs_now();
525	long diff = now - last;
526	last = now;
527	return diff;
528	}
529
530	static char*
531	main_format_bcnt (xoff_t r, char *buf)
532	{
533	static const char* fmts[] = { "B", "KB", "MB", "GB" };
534	int i;
535
536	for (i = 0; i < SIZEOF_ARRAY(fmts); i += 1)
537	{
538	if (r <= (10 * 1024) \|\| i == (-1 + (int)SIZEOF_ARRAY(fmts)))
539	{
540	sprintf (buf, "%"Q"u %s", r, fmts[i]);
541	break;
542	}
543	r /= 1024;
544	}
545	return buf;
546	}
547
548	static char*
549	main_format_rate (xoff_t bytes, long millis, char *buf)
550	{
551	xoff_t r = (xoff_t)(1.0 * bytes / (1.0 * millis / 1000.0));
552	static char lbuf[32];
553
554	main_format_bcnt (r, lbuf);
555	sprintf (buf, "%s/sec", lbuf);
556	return buf;
557	}
558
559	static char*
560	main_format_millis (long millis, char *buf)
561	{
562	if (millis < 1000) { sprintf (buf, "%lu ms", millis); }
563	else if (millis < 10000) { sprintf (buf, "%.1f sec", millis / 1000.0); }
564	else { sprintf (buf, "%lu sec", millis / 1000L); }
565	return buf;
566	}
567
568	/* A safe version of strtol for xoff_t. */
569	static int
570	main_strtoxoff (const char* s, xoff_t *xo, char which)
571	{
572	char *e;
573	xoff_t x;
574
575	XD3_ASSERT(s && *s != 0);
576
577	{
578	/* Should check LONG_MIN, LONG_MAX, LLONG_MIN, LLONG_MAX? */
579	#if SIZEOF_XOFF_T == 4
580	long xx = strtol (s, &e, 0);
581	#else
582	long long xx = strtoll (s, &e, 0);
583	#endif
584
585	if (xx < 0)
586	{
587	XPR(NT "-%c: negative integer: %s\n", which, s);
588	return EXIT_FAILURE;
589	}
590
591	x = xx;
592	}
593
594	if (*e != 0)
595	{
596	XPR(NT "-%c: invalid integer: %s\n", which, s);
597	return EXIT_FAILURE;
598	}
599
600	(*xo) = x;
601	return 0;
602	}
603
604	static int
605	main_atou (const char* arg, usize_t *xo, usize_t low, usize_t high, char which)
606	{
607	xoff_t x;
608	int ret;
609
610	if ((ret = main_strtoxoff (arg, & x, which))) { return ret; }
611
612	if (x < low)
613	{
614	XPR(NT "-%c: minimum value: %u\n", which, low);
615	return EXIT_FAILURE;
616	}
617	if (high == 0)
618	{
619	high = USIZE_T_MAX;
620	}
621	if (x > high)
622	{
623	XPR(NT "-%c: maximum value: %u\n", which, high);
624	return EXIT_FAILURE;
625	}
626	(*xo) = (usize_t)x;
627	return 0;
628	}
629
630	/******************************************************************************************
631	FILE BASICS
632	******************************************************************************************/
633
634	/* With all the variation in file system-call semantics, arguments, return values and
635	* error-handling for the POSIX and STDIO file APIs, the insides of these functions make
636	* me sick, which is why these wrappers exist. */
637
638	#define XOPEN_OPNAME (xfile->mode == XO_READ ? "read" : "write")
639	#define XOPEN_STDIO (xfile->mode == XO_READ ? "rb" : "wb")
640	#define XOPEN_POSIX (xfile->mode == XO_READ ? O_RDONLY : O_WRONLY \| O_CREAT \| O_TRUNC)
641	#define XOPEN_MODE (xfile->mode == XO_READ ? 0 : 0666)
642
643	#define XF_ERROR(op, name, ret) \
644	XPR(NT "file %s failed: %s: %s: %s\n", (op), XOPEN_OPNAME, (name), xd3_mainerror (ret))
645
646	#if XD3_STDIO
647	#define XFNO(f) fileno(f->file)
648	#define XSTDOUT_XF(f) { (f)->file = stdout; (f)->filename = "/dev/stdout"; }
649	#define XSTDIN_XF(f) { (f)->file = stdin; (f)->filename = "/dev/stdin"; }
650
651	#elif XD3_POSIX
652	#define XFNO(f) f->file
653	#define XSTDOUT_XF(f) { (f)->file = STDOUT_FILENO; (f)->filename = "/dev/stdout"; }
654	#define XSTDIN_XF(f) { (f)->file = STDIN_FILENO; (f)->filename = "/dev/stdin"; }
655
656	#elif XD3_WIN32
657	#define XFNO(f) -1
658	#define XSTDOUT_XF(f) { (f)->file = winStartupInfo.hStdOutput; (f)->filename = "(stdout)"; }
659	#define XSTDIN_XF(f) { (f)->file = winStartupInfo.hStdInput; (f)->filename = "(stdin)"; }
660	#endif
661
662	static void
663	main_file_init (main_file *xfile)
664	{
665	memset (xfile, 0, sizeof (*xfile));
666
667	#if XD3_POSIX
668	xfile->file = -1;
669	#endif
670	#if XD3_WIN32
671	xfile->file = INVALID_HANDLE_VALUE;
672	#endif
673	}
674
675	static int
676	main_file_isopen (main_file *xfile)
677	{
678	#if XD3_STDIO
679	return xfile->file != NULL;
680
681	#elif XD3_POSIX
682	return xfile->file != -1;
683
684	#elif XD3_WIN32
685	return xfile->file != INVALID_HANDLE_VALUE;
686	#endif
687	}
688
689	static int
690	main_file_close (main_file *xfile)
691	{
692	int ret = 0;
693
694	if (! main_file_isopen (xfile))
695	{
696	return 0;
697	}
698
699	#if XD3_STDIO
700	ret = fclose (xfile->file);
701	xfile->file = NULL;
702
703	#elif XD3_POSIX
704	ret = close (xfile->file);
705	xfile->file = -1;
706
707	#elif XD3_WIN32
708	if (!CloseHandle(xfile->file)) {
709	ret = get_errno ();
710	}
711	xfile->file = INVALID_HANDLE_VALUE;
712	#endif
713
714	if (ret != 0) { XF_ERROR ("close", xfile->filename, ret = get_errno ()); }
715	return ret;
716	}
717
718	static void
719	main_file_cleanup (main_file *xfile)
720	{
721	if (main_file_isopen (xfile))
722	{
723	main_file_close (xfile);
724	}
725
726	if (xfile->snprintf_buf != NULL)
727	{
728	main_free(xfile->snprintf_buf);
729	xfile->snprintf_buf = NULL;
730	}
731
732	if (xfile->filename_copy != NULL)
733	{
734	main_free(xfile->filename_copy);
735	xfile->filename_copy = NULL;
736	}
737	}
738
739	static int
740	main_file_open (main_file xfile, const char name, int mode)
741	{
742	int ret = 0;
743
744	xfile->mode = mode;
745
746	XD3_ASSERT (name != NULL);
747	XD3_ASSERT (! main_file_isopen (xfile));
748	if (name[0] == 0)
749	{
750	XPR(NT "invalid file name: empty string\n");
751	return XD3_INVALID;
752	}
753
754	#if XD3_STDIO
755	xfile->file = fopen (name, XOPEN_STDIO);
756
757	ret = (xfile->file == NULL) ? get_errno () : 0;
758
759	#elif XD3_POSIX
760	if ((ret = open (name, XOPEN_POSIX, XOPEN_MODE)) < 0)
761	{
762	ret = get_errno ();
763	}
764	else
765	{
766	xfile->file = ret;
767	ret = 0;
768	}
769
770	#elif XD3_WIN32
771	xfile->file = CreateFile(name,
772	(mode == XO_READ) ? GENERIC_READ : GENERIC_WRITE,
773	FILE_SHARE_READ,
774	NULL,
775	(mode == XO_READ) ? OPEN_EXISTING : (option_force ? CREATE_ALWAYS : CREATE_NEW),
776	FILE_ATTRIBUTE_NORMAL,
777	NULL);
778	if (xfile->file == INVALID_HANDLE_VALUE) {
779	ret = get_errno ();
780	}
781	#endif
782	if (ret) { XF_ERROR ("open", name, ret); }
783	else { xfile->realname = name; xfile->nread = 0; }
784	return ret;
785	}
786
787	static int
788	main_file_stat (main_file xfile, xoff_t size, int err_ifnoseek)
789	{
790	int ret = 0;
791	#if XD3_WIN32
792	LARGE_INTEGER li;
793	if (GetFileSizeEx(xfile->file, &li) == 0) {
794	ret = get_errno ();
795	} else {
796	*size = li.QuadPart;
797	}
798	#else
799	struct stat sbuf;
800	if (fstat (XFNO (xfile), & sbuf) < 0)
801	{
802	ret = get_errno ();
803	if (err_ifnoseek) { XF_ERROR ("stat", xfile->filename, ret); }
804	return ret;
805	}
806
807	if (! S_ISREG (sbuf.st_mode))
808	{
809	if (err_ifnoseek) { XPR(NT "source file must be seekable: %s\n", xfile->filename); }
810	return ESPIPE;
811	}
812	(*size) = sbuf.st_size;
813	#endif
814	return ret;
815	}
816
817	static int
818	main_file_exists (main_file *xfile)
819	{
820	struct stat sbuf;
821	return stat (xfile->filename, & sbuf) == 0 && S_ISREG (sbuf.st_mode);
822	}
823
824	#if (XD3_POSIX \|\| EXTERNAL_COMPRESSION)
825	/* POSIX-generic code takes a function pointer to read() or write(). This calls the
826	* function repeatedly until the buffer is full or EOF. The NREAD parameter is not
827	* set for write, NULL is passed. Return is signed, < 0 indicate errors, otherwise
828	* byte count. */
829	typedef int (xd3_posix_func) (int fd, uint8_t *buf, usize_t size);
830
831	static int
832	xd3_posix_io (int fd, uint8_t buf, usize_t size, xd3_posix_func func, usize_t *nread)
833	{
834	int ret;
835	usize_t nproc = 0;
836
837	while (nproc < size)
838	{
839	int result = (*func) (fd, buf + nproc, size - nproc);
840
841	if (result < 0)
842	{
843	ret = get_errno ();
844	if (ret != EAGAIN && ret != EINTR)
845	{
846	return ret;
847	}
848	result = 0;
849	}
850
851	if (nread != NULL && result == 0) { break; }
852
853	nproc += result;
854	}
855	if (nread != NULL) { (*nread) = nproc; }
856	return 0;
857	}
858	#endif
859
860	/* POSIX is unbuffered, while STDIO is buffered. main_file_read() should always be called
861	* on blocks. */
862	static int
863	main_file_read (main_file *ifile,
864	uint8_t *buf,
865	usize_t size,
866	usize_t *nread,
867	const char *msg)
868	{
869	int ret = 0;
870
871	#if XD3_STDIO
872	usize_t result;
873
874	result = fread (buf, 1, size, ifile->file);
875
876	if (result < size && ferror (ifile->file))
877	{
878	ret = get_errno ();
879	}
880	else
881	{
882	*nread = result;
883	}
884
885	#elif XD3_POSIX
886	ret = xd3_posix_io (ifile->file, buf, size, (xd3_posix_func*) &read, nread);
887
888	#elif XD3_WIN32
889	DWORD nread2;
890	if (ReadFile (ifile->file, buf, size, &nread2, NULL) == 0) {
891	ret = get_errno();
892	} else {
893	*nread = (usize_t)nread2;
894	}
895	#endif
896
897	if (ret)
898	{
899	XPR(NT "%s: %s: %s\n", msg, ifile->filename, xd3_mainerror (ret));
900	}
901	else
902	{
903	if (option_verbose > 3) { XPR(NT "main read: %s: %u\n", ifile->filename, (*nread)); }
904	ifile->nread += (*nread);
905	}
906
907	return ret;
908	}
909
910	static int
911	main_file_write (main_file ofile, uint8_t buf, usize_t size, const char *msg)
912	{
913	int ret = 0;
914
915	#if XD3_STDIO
916	usize_t result;
917
918	result = fwrite (buf, 1, size, ofile->file);
919
920	if (result != size) { ret = get_errno (); }
921
922	#elif XD3_POSIX
923	ret = xd3_posix_io (ofile->file, buf, size, (xd3_posix_func*) &write, NULL);
924
925	#elif XD3_WIN32
926	DWORD nwrite;
927	if (WriteFile(ofile->file, buf, size, &nwrite, NULL) == 0) {
928	ret = get_errno ();
929	} else {
930	if (size != nwrite) {
931	XPR(NT "Incorrect write count");
932	ret = XD3_INTERNAL;
933	}
934	}
935	#endif
936
937	if (ret)
938	{
939	XPR(NT "%s: %s: %s\n", msg, ofile->filename, xd3_mainerror (ret));
940	}
941	else
942	{
943	if (option_verbose > 3) { XPR(NT "main write: %s: %u\n", ofile->filename, size); }
944	ofile->nwrite += size;
945	}
946
947	return ret;
948	}
949
950	static int
951	main_file_seek (main_file *xfile, xoff_t pos)
952	{
953	int ret = 0;
954
955	#if XD3_STDIO
956	if (fseek (xfile->file, pos, SEEK_SET) != 0) { ret = get_errno (); }
957
958	#elif XD3_POSIX
959	if (lseek (xfile->file, pos, SEEK_SET) != pos) { ret = get_errno (); }
960
961	#elif XD3_WIN32
962	LARGE_INTEGER move, out;
963	move.QuadPart = pos;
964	if (SetFilePointerEx(xfile->file, move, &out, FILE_BEGIN) == 0) {
965	ret = get_errno ();
966	}
967	#endif
968
969	if (ret)
970	{
971	XPR(NT "seek failed: %s: %s\n", xfile->filename, xd3_mainerror (ret));
972	}
973
974	return ret;
975	}
976
977	/******************************************************************************************
978	VCDIFF TOOLS
979	******************************************************************************************/
980
981	#if VCDIFF_TOOLS
982
983	/* The following macros let VCDIFF printing something printf-like with
984	* main_file_write(), e.g.,:
985	*
986	* VC(UT "trying to be portable: %d\n", x)VE;
987	*/
988	#define SNPRINTF_BUFSIZE 1024
989	#define VC do { if (((ret = snprintf
990	#define UT (char*)xfile->snprintf_buf, SNPRINTF_BUFSIZE,
991	#define VE ) >= SNPRINTF_BUFSIZE \
992	&& (ret = main_print_overflow(ret)) != 0) \
993	\|\| (ret = main_file_write(xfile, xfile->snprintf_buf, \
994	ret, "print")) != 0) \
995	{ return ret; } } while (0)
996
997	#ifdef WIN32
998	/* According to the internet, Windows vsnprintf() differs from most Unix
999	* implementations regarding the terminating 0 when the boundary condition
1000	* is met. It doesn't matter here, we don't rely on the trailing 0. */
1001	#include <stdarg.h>
1002	int
1003	snprintf (char str, int n, char fmt, ...)
1004	{
1005	va_list a;
1006	int ret;
1007	va_start (a, fmt);
1008	ret = vsnprintf (str, n, fmt, a);
1009	va_end (a);
1010	return ret;
1011	}
1012	#endif
1013
1014	static int
1015	main_print_overflow (int x)
1016	{
1017	XPR(NT "internal print buffer overflow: %d bytes\n", x);
1018	return XD3_INTERNAL;
1019	}
1020
1021	/* This function prints a single VCDIFF window. */
1022	static int
1023	main_print_window (xd3_stream* stream, main_file *xfile)
1024	{
1025	int ret;
1026	usize_t size = 0;
1027
1028	VC(UT " Offset Code Type1 Size1 @Addr1 + Type2 Size2 @Addr2\n")VE;
1029
1030	while (stream->inst_sect.buf < stream->inst_sect.buf_max)
1031	{
1032	uint code = stream->inst_sect.buf[0];
1033
1034	if ((ret = xd3_decode_instruction (stream))) { return ret; }
1035
1036	VC(UT " %06"Q"u %03u %s %3u", stream->dec_winstart + size, code,
1037	xd3_rtype_to_string (stream->dec_current1.type, option_print_cpymode),
1038	(usize_t) stream->dec_current1.size)VE;
1039
1040	if (stream->dec_current1.type != XD3_NOOP)
1041	{
1042	size += stream->dec_current1.size;
1043	if (stream->dec_current1.type >= XD3_CPY)
1044	{
1045	VC(UT " @%-6u", (usize_t)stream->dec_current1.addr)VE;
1046	}
1047	else
1048	{
1049	VC(UT " ")VE;
1050	}
1051	}
1052
1053	if (stream->dec_current2.type != XD3_NOOP)
1054	{
1055	size += stream->dec_current2.size;
1056	VC(UT " %s %3u",
1057	xd3_rtype_to_string (stream->dec_current2.type, option_print_cpymode),
1058	(usize_t)stream->dec_current2.size)VE;
1059
1060	if (stream->dec_current2.type >= XD3_CPY)
1061	{
1062	VC(UT " @%-6u", (usize_t)stream->dec_current2.addr)VE;
1063	}
1064	}
1065
1066	VC(UT "\n")VE;
1067	}
1068
1069	if (stream->dec_tgtlen != size && (stream->flags & XD3_SKIP_WINDOW) == 0)
1070	{
1071	XPR(NT "target window size inconsistency");
1072	return XD3_INTERNAL;
1073	}
1074
1075	if (stream->dec_position != stream->dec_maxpos)
1076	{
1077	XPR(NT "target window position inconsistency");
1078	return XD3_INTERNAL;
1079	}
1080
1081	if (stream->addr_sect.buf != stream->addr_sect.buf_max)
1082	{
1083	XPR(NT "address section inconsistency");
1084	return XD3_INTERNAL;
1085	}
1086
1087	return 0;
1088	}
1089
1090	static int
1091	main_print_vcdiff_file (main_file xfile, main_file file, const char *type)
1092	{
1093	int ret; /* Used by above macros */
1094	if (file->filename)
1095	{
1096	VC(UT "XDELTA filename (%s): %s\n", type, file->filename)VE;
1097	}
1098	if (file->compressor)
1099	{
1100	VC(UT "XDELTA ext comp (%s): %s\n", type, file->compressor->recomp_cmdname)VE;
1101	}
1102	return 0;
1103	}
1104
1105	/* This function prints a VCDIFF input, mainly for debugging purposes. */
1106	static int
1107	main_print_func (xd3_stream* stream, main_file *xfile)
1108	{
1109	int ret;
1110
1111	if (xfile->snprintf_buf == NULL)
1112	{
1113	if ((xfile->snprintf_buf = main_malloc(SNPRINTF_BUFSIZE)) == NULL)
1114	{
1115	return ENOMEM;
1116	}
1117	}
1118
1119	if (stream->dec_winstart == 0)
1120	{
1121	VC(UT "VCDIFF version: 0\n")VE;
1122
1123	VC(UT "VCDIFF header size: %d\n", stream->dec_hdrsize)VE;
1124	VC(UT "VCDIFF header indicator: ")VE;
1125	if ((stream->dec_hdr_ind & VCD_SECONDARY) != 0) VC(UT "VCD_SECONDARY ")VE;
1126	if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0) VC(UT "VCD_CODETABLE ")VE;
1127	if ((stream->dec_hdr_ind & VCD_APPHEADER) != 0) VC(UT "VCD_APPHEADER ")VE;
1128	if (stream->dec_hdr_ind == 0) VC(UT "none")VE;
1129	VC(UT "\n")VE;
1130
1131	IF_SEC(VC(UT "VCDIFF secondary compressor: %s\n",
1132	stream->sec_type ? stream->sec_type->name : "none")VE);
1133	IF_NSEC(VC(UT "VCDIFF secondary compressor: unsupported\n")VE);
1134
1135	if (stream->dec_hdr_ind & VCD_APPHEADER)
1136	{
1137	uint8_t *apphead;
1138	usize_t appheadsz;
1139	ret = xd3_get_appheader (stream, & apphead, & appheadsz);
1140
1141	if (ret == 0 && appheadsz > 0)
1142	{
1143	int sq = option_quiet;
1144	main_file i, o, s;
1145	XD3_ASSERT (apphead != NULL);
1146	VC(UT "VCDIFF application header: ")VE;
1147	if ((ret = main_file_write (xfile, apphead, appheadsz, "print")) != 0) { return ret; }
1148	VC(UT "\n")VE;
1149
1150	main_file_init (& i);
1151	main_file_init (& o);
1152	main_file_init (& s);
1153	option_quiet = 1;
1154	main_get_appheader (stream, &i, & o, & s);
1155	option_quiet = sq;
1156	if ((ret = main_print_vcdiff_file (xfile, & o, "output"))) { return ret; }
1157	if ((ret = main_print_vcdiff_file (xfile, & s, "source"))) { return ret; }
1158	main_file_cleanup (& i);
1159	main_file_cleanup (& o);
1160	main_file_cleanup (& s);
1161	}
1162	}
1163	}
1164	else
1165	{
1166	VC(UT "\n")VE;
1167	}
1168
1169	VC(UT "VCDIFF window number: %"Q"u\n", stream->current_window)VE;
1170	VC(UT "VCDIFF window indicator: ")VE;
1171	if ((stream->dec_win_ind & VCD_SOURCE) != 0) VC(UT "VCD_SOURCE ")VE;
1172	if ((stream->dec_win_ind & VCD_TARGET) != 0) VC(UT "VCD_TARGET ")VE;
1173	if ((stream->dec_win_ind & VCD_ADLER32) != 0) VC(UT "VCD_ADLER32 ")VE;
1174	if (stream->dec_win_ind == 0) VC(UT "none")VE;
1175	VC(UT "\n")VE;
1176
1177	if ((stream->dec_win_ind & VCD_ADLER32) != 0)
1178	{
1179	VC(UT "VCDIFF adler32 checksum: %08X\n", (usize_t)stream->dec_adler32)VE;
1180	}
1181
1182	if (stream->dec_del_ind != 0)
1183	{
1184	VC(UT "VCDIFF delta indicator: ")VE;
1185	if ((stream->dec_del_ind & VCD_DATACOMP) != 0) VC(UT "VCD_DATACOMP ")VE;
1186	if ((stream->dec_del_ind & VCD_INSTCOMP) != 0) VC(UT "VCD_INSTCOMP ")VE;
1187	if ((stream->dec_del_ind & VCD_ADDRCOMP) != 0) VC(UT "VCD_ADDRCOMP ")VE;
1188	if (stream->dec_del_ind == 0) VC(UT "none")VE;
1189	VC(UT "\n")VE;
1190	}
1191
1192	if (stream->dec_winstart != 0)
1193	{
1194	VC(UT "VCDIFF window at offset: %"Q"u\n", stream->dec_winstart)VE;
1195	}
1196
1197	if (SRCORTGT (stream->dec_win_ind))
1198	{
1199	VC(UT "VCDIFF copy window length: %u\n", (usize_t)stream->dec_cpylen)VE;
1200	VC(UT "VCDIFF copy window offset: %"Q"u\n", stream->dec_cpyoff)VE;
1201	}
1202
1203	VC(UT "VCDIFF delta encoding length: %u\n", (usize_t)stream->dec_enclen)VE;
1204	VC(UT "VCDIFF target window length: %u\n", (usize_t)stream->dec_tgtlen)VE;
1205
1206	VC(UT "VCDIFF data section length: %u\n", (usize_t)stream->data_sect.size)VE;
1207	VC(UT "VCDIFF inst section length: %u\n", (usize_t)stream->inst_sect.size)VE;
1208	VC(UT "VCDIFF addr section length: %u\n", (usize_t)stream->addr_sect.size)VE;
1209
1210	ret = 0;
1211	if ((stream->flags & XD3_JUST_HDR) != 0)
1212	{
1213	/* Print a header -- finished! */
1214	ret = PRINTHDR_SPECIAL;
1215	}
1216	else if ((stream->flags & XD3_SKIP_WINDOW) == 0)
1217	{
1218	ret = main_print_window (stream, xfile);
1219	}
1220
1221	return ret;
1222	}
1223	#endif /* VCDIFF_TOOLS */
1224
1225	/******************************************************************************************
1226	Input decompression, output recompression
1227	******************************************************************************************/
1228
1229	#if EXTERNAL_COMPRESSION
1230	/* This is tricky POSIX-specific code with lots of fork(), pipe(), dup(), waitpid(), and
1231	* exec() business. Most of this code originated in PRCS1, which did automatic
1232	* package-file decompression. It works with both XD3_POSIX and XD3_STDIO file
1233	* disciplines.
1234	*
1235	* To automatically detect compressed inputs requires a child process to reconstruct the
1236	* input stream, which was advanced in order to detect compression, because it may not be
1237	* seekable. In other words, the main program reads part of the input stream, and if it
1238	* detects a compressed input it then forks a pipe copier process, which copies the
1239	* first-read block out of the main-program's memory, then streams the remaining
1240	* compressed input into the input-decompression pipe.
1241	*/
1242
1243	#include <unistd.h>
1244	#include <sys/stat.h>
1245	#include <sys/wait.h>
1246
1247	/* Remember which pipe FD is which. */
1248	#define PIPE_READ_FD 0
1249	#define PIPE_WRITE_FD 1
1250
1251	static pid_t ext_subprocs[2];
1252	static char* ext_tmpfile = NULL;
1253
1254	/* Like write(), but makes repeated calls to empty the buffer. */
1255	static int
1256	main_pipe_write (int outfd, const uint8_t *exist_buf, usize_t remain)
1257	{
1258	int ret;
1259
1260	if ((ret = xd3_posix_io (outfd, (uint8_t) exist_buf, remain, (xd3_posix_func) &write, NULL)))
1261	{
1262	XPR(NT "pipe write failed: %s", xd3_mainerror (ret));
1263	return ret;
1264	}
1265
1266	return 0;
1267	}
1268
1269	/* A simple error-reporting waitpid interface. */
1270	static int
1271	main_waitpid_check(pid_t pid)
1272	{
1273	int status;
1274	int ret = 0;
1275
1276	if (waitpid (pid, & status, 0) < 0)
1277	{
1278	ret = get_errno ();
1279	XPR(NT "compression subprocess: wait: %s\n", xd3_mainerror (ret));
1280	}
1281	else if (! WIFEXITED (status))
1282	{
1283	ret = ECHILD;
1284	XPR(NT "compression subprocess: signal %d\n",
1285	WIFSIGNALED (status) ? WTERMSIG (status) : WSTOPSIG (status));
1286	}
1287	else if (WEXITSTATUS (status) != 0)
1288	{
1289	ret = ECHILD;
1290	XPR(NT "compression subprocess: exit %d\n", WEXITSTATUS (status));
1291	}
1292
1293	return ret;
1294	}
1295
1296	/* Wait for any existing child processes to check for abnormal exit. */
1297	static int
1298	main_external_compression_finish (void)
1299	{
1300	int i;
1301	int ret;
1302
1303	for (i = 0; i < 2; i += 1)
1304	{
1305	if (! ext_subprocs[i]) { continue; }
1306
1307	if ((ret = main_waitpid_check (ext_subprocs[i])))
1308	{
1309	return ret;
1310	}
1311	}
1312
1313	return 0;
1314	}
1315
1316	/* This runs as a forked process of main_input_decompress_setup() to copy input to the
1317	* decompression process. First, the available input is copied out of the existing
1318	* buffer, then the buffer is reused to continue reading from the compressed input
1319	* file. */
1320	static int
1321	main_pipe_copier (uint8_t *pipe_buf,
1322	usize_t pipe_bufsize,
1323	usize_t nread,
1324	main_file *ifile,
1325	int outfd)
1326	{
1327	int ret;
1328
1329	for (;;)
1330	{
1331	if (nread > 0 && (ret = main_pipe_write (outfd, pipe_buf, nread)))
1332	{
1333	return ret;
1334	}
1335
1336	if (nread < pipe_bufsize)
1337	{
1338	break;
1339	}
1340
1341	if ((ret = main_file_read (ifile, pipe_buf, pipe_bufsize, & nread, "pipe read failed")) < 0)
1342	{
1343	return ret;
1344	}
1345	}
1346
1347	return 0;
1348	}
1349
1350	/* This function is called after we have read some amount of data from the input file and
1351	* detected a compressed input. Here we start a decompression subprocess by forking
1352	* twice. The first process runs the decompression command, the second process copies
1353	* data to the input of the first. */
1354	static int
1355	main_input_decompress_setup (const main_extcomp *decomp,
1356	main_file *ifile,
1357	uint8_t *input_buf,
1358	usize_t input_bufsize,
1359	uint8_t *pipe_buf,
1360	usize_t pipe_bufsize,
1361	usize_t pipe_avail,
1362	usize_t *nread)
1363	{
1364	int outpipefd[2], inpipefd[2]; /* The two pipes: input and output file descriptors. */
1365	int input_fd = -1; /* The resulting input_fd (output of decompression). */
1366	pid_t decomp_id, copier_id; /* The two subprocs. */
1367	int ret;
1368
1369	outpipefd[0] = outpipefd[1] = -1;
1370	inpipefd[0] = inpipefd[1] = -1;
1371
1372	if (pipe (outpipefd) \|\| pipe (inpipefd))
1373	{
1374	XPR(NT "pipe failed: %s\n", xd3_mainerror (ret = get_errno ()));
1375	goto pipe_cleanup;
1376	}
1377
1378	if ((decomp_id = fork ()) < 0)
1379	{
1380	XPR(NT "fork failed: %s\n", xd3_mainerror (ret = get_errno ()));
1381	goto pipe_cleanup;
1382	}
1383
1384	/* The first child runs the decompression process: */
1385	if (decomp_id == 0)
1386	{
1387	/* Setup pipes: write to the outpipe, read from the inpipe. */
1388	if (dup2 (outpipefd[PIPE_WRITE_FD], STDOUT_FILENO) < 0 \|\|
1389	dup2 (inpipefd[PIPE_READ_FD], STDIN_FILENO) < 0 \|\|
1390	close (outpipefd[PIPE_READ_FD]) \|\|
1391	close (outpipefd[PIPE_WRITE_FD]) \|\|
1392	close (inpipefd[PIPE_READ_FD]) \|\|
1393	close (inpipefd[PIPE_WRITE_FD]) \|\|
1394	execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, decomp->decomp_options, NULL))
1395	{
1396	XPR(NT "child process %s failed to execute: %s\n", decomp->decomp_cmdname, xd3_mainerror (get_errno ()));
1397	}
1398
1399	_exit (127);
1400	}
1401
1402	ext_subprocs[0] = decomp_id;
1403
1404	if ((copier_id = fork ()) < 0)
1405	{
1406	XPR(NT "fork failed: %s\n", xd3_mainerror (ret = get_errno ()));
1407	goto pipe_cleanup;
1408	}
1409
1410	/* The second child runs the copier process: */
1411	if (copier_id == 0)
1412	{
1413	int exitval = 0;
1414
1415	if (close (inpipefd[PIPE_READ_FD]) \|\|
1416	main_pipe_copier (pipe_buf, pipe_bufsize, pipe_avail, ifile, inpipefd[PIPE_WRITE_FD]) \|\|
1417	close (inpipefd[PIPE_WRITE_FD]))
1418	{
1419	XPR(NT "child copier process failed: %s\n", xd3_mainerror (get_errno ()));
1420	exitval = 1;
1421	}
1422
1423	_exit (exitval);
1424	}
1425
1426	ext_subprocs[1] = copier_id;
1427
1428	/* The parent closes both pipes after duplicating the output of compression. */
1429	input_fd = dup (outpipefd[PIPE_READ_FD]);
1430
1431	if (input_fd < 0 \|\|
1432	main_file_close (ifile) \|\|
1433	close (outpipefd[PIPE_READ_FD]) \|\|
1434	close (outpipefd[PIPE_WRITE_FD]) \|\|
1435	close (inpipefd[PIPE_READ_FD]) \|\|
1436	close (inpipefd[PIPE_WRITE_FD]))
1437	{
1438	XPR(NT "dup/close failed: %s\n", xd3_mainerror (ret = get_errno ()));
1439	goto pipe_cleanup;
1440	}
1441
1442	#if XD3_STDIO
1443	/* Note: fdopen() acquires the fd, closes it when finished. */
1444	if ((ifile->file = fdopen (input_fd, "r")) == NULL)
1445	{
1446	XPR(NT "fdopen failed: %s\n", xd3_mainerror (ret = get_errno ()));
1447	goto pipe_cleanup;
1448	}
1449
1450	#elif XD3_POSIX
1451	ifile->file = input_fd;
1452	#endif
1453
1454	ifile->compressor = decomp;
1455
1456	/* Now the input file is decompressed. */
1457	return main_file_read (ifile, input_buf, input_bufsize, nread, "input decompression failed");
1458
1459	pipe_cleanup:
1460	close (input_fd);
1461	close (outpipefd[PIPE_READ_FD]);
1462	close (outpipefd[PIPE_WRITE_FD]);
1463	close (inpipefd[PIPE_READ_FD]);
1464	close (inpipefd[PIPE_WRITE_FD]);
1465	return ret;
1466	}
1467
1468
1469	/* This routine is called when the first buffer of input data is read by the main program
1470	* (unless input decompression is disabled by command-line option). If it recognizes the
1471	* magic number of a known input type it invokes decompression.
1472	*
1473	* Skips decompression if the decompression type or the file type is RD_NONEXTERNAL.
1474	*
1475	* Behaves exactly like main_file_read, otherwise.
1476	*
1477	* This function uses a separate buffer to read the first small block of input. If a
1478	* compressed input is detected, the separate buffer is passed to the pipe copier. This
1479	* avoids using the same size buffer in both cases. */
1480	static int
1481	main_decompress_input_check (main_file *ifile,
1482	uint8_t *input_buf,
1483	usize_t input_size,
1484	usize_t *nread)
1485	{
1486	int i;
1487	int ret;
1488	uint8_t check_buf[XD3_ALLOCSIZE];
1489	usize_t check_nread;
1490
1491	if ((ret = main_file_read (ifile, check_buf, min (input_size, XD3_ALLOCSIZE), & check_nread, "input read failed")))
1492	{
1493	return ret;
1494	}
1495
1496	for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)
1497	{
1498	const main_extcomp *decomp = & extcomp_types[i];
1499
1500	if ((check_nread > decomp->magic_size) &&
1501	/* The following expr skips decompression if we are trying to read a VCDIFF
1502	* input and that is the magic number. */
1503	!((decomp->flags & RD_NONEXTERNAL) && (ifile->flags & RD_NONEXTERNAL)) &&
1504	memcmp (check_buf, decomp->magic, decomp->magic_size) == 0)
1505	{
1506	if (! option_quiet)
1507	{
1508	XPR(NT "%s \| %s %s\n",
1509	ifile->filename,
1510	decomp->decomp_cmdname,
1511	decomp->decomp_options);
1512	}
1513
1514	return main_input_decompress_setup (decomp, ifile,
1515	input_buf, input_size,
1516	check_buf, XD3_ALLOCSIZE,
1517	check_nread, nread);
1518	}
1519	}
1520
1521	/* Now read the rest of the input block. */
1522	(*nread) = 0;
1523
1524	if (check_nread == XD3_ALLOCSIZE)
1525	{
1526	ret = main_file_read (ifile, input_buf + XD3_ALLOCSIZE,
1527	input_size - XD3_ALLOCSIZE, nread,
1528	"input read failed");
1529	}
1530
1531	memcpy (input_buf, check_buf, check_nread);
1532
1533	(*nread) += check_nread;
1534
1535	return 0;
1536	}
1537
1538	/* This is called when the source file needs to be decompressed. We fork/exec a
1539	* decompression command with the proper input and output to a temporary file. */
1540	static int
1541	main_decompress_source (main_file sfile, xd3_source source)
1542	{
1543	const main_extcomp *decomp = sfile->compressor;
1544	pid_t decomp_id; /* One subproc. */
1545	int input_fd = -1;
1546	int output_fd = -1;
1547	int ret;
1548	char *tmpname = NULL;
1549	char *tmpdir = getenv ("TMPDIR");
1550	static const char tmpl[] = "/xd3src.XXXXXX";
1551
1552	/* Make a template for mkstmp() */
1553	if (tmpdir == NULL) { tmpdir = "/tmp"; }
1554	if ((tmpname = main_malloc (strlen (tmpdir) + sizeof (tmpl) + 1)) == NULL) { return ENOMEM; }
1555	sprintf (tmpname, "%s%s", tmpdir, tmpl);
1556
1557	XD3_ASSERT (ext_tmpfile == NULL);
1558	ext_tmpfile = tmpname;
1559
1560	/* Open the output FD. */
1561	if ((output_fd = mkstemp (tmpname)) < 0)
1562	{
1563	XPR(NT "mkstemp failed: %s: %s", tmpname, xd3_mainerror (ret = get_errno ()));
1564	goto cleanup;
1565	}
1566
1567	/* Copy the input FD, reset file position. */
1568	XD3_ASSERT (main_file_isopen (sfile));
1569	#if XD3_STDIO
1570	if ((input_fd = dup (fileno (sfile->file))) < 0)
1571	{
1572	XPR(NT "dup failed: %s", xd3_mainerror (ret = get_errno ()));
1573	goto cleanup;
1574	}
1575	main_file_close (sfile);
1576	sfile->file = NULL;
1577	#elif XD3_POSIX
1578	input_fd = sfile->file;
1579	sfile->file = -1;
1580	#endif
1581
1582	if ((ret = lseek (input_fd, SEEK_SET, 0)) != 0)
1583	{
1584	XPR(NT "lseek failed: : %s", xd3_mainerror (ret = get_errno ()));
1585	goto cleanup;
1586	}
1587
1588	if ((decomp_id = fork ()) < 0)
1589	{
1590	XPR(NT "fork failed: %s", xd3_mainerror (ret = get_errno ()));
1591	goto cleanup;
1592	}
1593
1594	/* The child runs the decompression process: */
1595	if (decomp_id == 0)
1596	{
1597	/* Setup pipes: write to the output file, read from the pipe. */
1598	if (dup2 (input_fd, STDIN_FILENO) < 0 \|\|
1599	dup2 (output_fd, STDOUT_FILENO) < 0 \|\|
1600	execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, decomp->decomp_options, NULL))
1601	{
1602	XPR(NT "child process %s failed to execute: %s\n",
1603	decomp->decomp_cmdname, xd3_mainerror (get_errno ()));
1604	}
1605
1606	_exit (127);
1607	}
1608
1609	close (input_fd);
1610	close (output_fd);
1611	input_fd = -1;
1612	output_fd = -1;
1613
1614	/* Then wait for completion. */
1615	if ((ret = main_waitpid_check (decomp_id)))
1616	{
1617	goto cleanup;
1618	}
1619
1620	/* Open/stat the decompressed source file. */
1621	if ((ret = main_file_open (sfile, tmpname, XO_READ))) { goto cleanup; }
1622	if ((ret = main_file_stat (sfile, & source->size, 1))) { goto cleanup; }
1623	return 0;
1624
1625	cleanup:
1626	close (input_fd);
1627	close (output_fd);
1628	if (tmpname) { free (tmpname); }
1629	ext_tmpfile = NULL;
1630	return ret;
1631	}
1632
1633	/* Initiate re-compression of the output stream. This is easier than input decompression
1634	* because we know beforehand that the stream will be compressed, whereas the input has
1635	* already been read when we decide it should be decompressed. Thus, it only requires one
1636	* subprocess and one pipe. */
1637	static int
1638	main_recompress_output (main_file *ofile)
1639	{
1640	pid_t recomp_id; /* One subproc. */
1641	int pipefd[2]; /* One pipe. */
1642	int output_fd = -1;
1643	int ret;
1644	const main_extcomp *recomp = ofile->compressor;
1645
1646	pipefd[0] = pipefd[1] = -1;
1647
1648	if (pipe (pipefd))
1649	{
1650	XPR(NT "pipe failed: %s\n", xd3_mainerror (ret = get_errno ()));
1651	goto pipe_cleanup;
1652	}
1653
1654	if ((recomp_id = fork ()) < 0)
1655	{
1656	XPR(NT "fork failed: %s\n", xd3_mainerror (ret = get_errno ()));
1657	goto pipe_cleanup;
1658	}
1659
1660	/* The child runs the recompression process: */
1661	if (recomp_id == 0)
1662	{
1663	/* Setup pipes: write to the output file, read from the pipe. */
1664	if (dup2 (XFNO (ofile), STDOUT_FILENO) < 0 \|\|
1665	dup2 (pipefd[PIPE_READ_FD], STDIN_FILENO) < 0 \|\|
1666	close (pipefd[PIPE_READ_FD]) \|\|
1667	close (pipefd[PIPE_WRITE_FD]) \|\|
1668	execlp (recomp->recomp_cmdname, recomp->recomp_cmdname, recomp->recomp_options, NULL))
1669	{
1670	XPR(NT "child process %s failed to execute: %s\n", recomp->recomp_cmdname, xd3_mainerror (get_errno ()));
1671	}
1672
1673	_exit (127);
1674	}
1675
1676	ext_subprocs[0] = recomp_id;
1677
1678	/* The parent closes both pipes after duplicating the output-fd for writing to the
1679	* compression pipe. */
1680	output_fd = dup (pipefd[PIPE_WRITE_FD]);
1681
1682	if (output_fd < 0 \|\|
1683	main_file_close (ofile) \|\|
1684	close (pipefd[PIPE_READ_FD]) \|\|
1685	close (pipefd[PIPE_WRITE_FD]))
1686	{
1687	XPR(NT "close failed: %s\n", xd3_mainerror (ret = get_errno ()));
1688	goto pipe_cleanup;
1689	}
1690
1691	#if XD3_STDIO
1692	/* Note: fdopen() acquires the fd, closes it when finished. */
1693	if ((ofile->file = fdopen (output_fd, "w")) == NULL)
1694	{
1695	XPR(NT "fdopen failed: %s\n", xd3_mainerror (ret = get_errno ()));
1696	goto pipe_cleanup;
1697	}
1698
1699	#elif XD3_POSIX
1700	ofile->file = output_fd;
1701	#endif
1702
1703	/* Now the output file will be compressed. */
1704	return 0;
1705
1706	pipe_cleanup:
1707	close (output_fd);
1708	close (pipefd[PIPE_READ_FD]);
1709	close (pipefd[PIPE_WRITE_FD]);
1710	return ret;
1711	}
1712	#endif /* EXTERNAL_COMPRESSION */
1713
1714	/* Identify the compressor that was used based on its ident string, which is passed in the
1715	* application header. */
1716	static const main_extcomp*
1717	main_ident_compressor (const char *ident)
1718	{
1719	int i;
1720
1721	for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)
1722	{
1723	if (strcmp (extcomp_types[i].ident, ident) == 0)
1724	{
1725	return & extcomp_types[i];
1726	}
1727	}
1728
1729	return NULL;
1730	}
1731
1732	/* Return the main_extcomp record to use for this identifier, if possible. */
1733	static const main_extcomp*
1734	main_get_compressor (const char *ident)
1735	{
1736	const main_extcomp *ext = main_ident_compressor (ident);
1737
1738	if (ext == NULL)
1739	{
1740	if (! option_quiet)
1741	{
1742	XPR(NT "warning: cannot recompress output: "
1743	"unrecognized external compression ID: %s\n", ident);
1744	}
1745	return NULL;
1746	}
1747	else if (! EXTERNAL_COMPRESSION)
1748	{
1749	if (! option_quiet)
1750	{
1751	XPR(NT "warning: external support not compiled: "
1752	"original input was compressed: %s\n", ext->recomp_cmdname);
1753	}
1754	return NULL;
1755	}
1756	else
1757	{
1758	return ext;
1759	}
1760	}
1761
1762	/******************************************************************************************
1763	APPLICATION HEADER
1764	******************************************************************************************/
1765
1766	#if XD3_ENCODER
1767	static const char*
1768	main_apphead_string (const char* x)
1769	{
1770	const char *y;
1771
1772	if (x == NULL) { return ""; }
1773
1774	if (strcmp (x, "/dev/stdin") == 0 \|\|
1775	strcmp (x, "/dev/stdout") == 0 \|\|
1776	strcmp (x, "/dev/stderr") == 0) { return "-"; }
1777
1778	// TODO: this is not portable
1779	return (y = strrchr (x, '/')) == NULL ? x : y + 1;
1780	}
1781
1782	static int
1783	main_set_appheader (xd3_stream stream, main_file input, main_file *sfile)
1784	{
1785	/* The user may disable the application header. Once the appheader is set, this
1786	* disables setting it again. */
1787	if (appheader_used \|\| ! option_use_appheader) { return 0; }
1788
1789	/* The user may specify the application header, otherwise format the default header. */
1790	if (option_appheader)
1791	{
1792	appheader_used = option_appheader;
1793	}
1794	else
1795	{
1796	const char *iname;
1797	const char *icomp;
1798	const char *sname;
1799	const char *scomp;
1800	int len;
1801
1802	iname = main_apphead_string (input->filename);
1803	icomp = (input->compressor == NULL) ? "" : input->compressor->ident;
1804	len = strlen (iname) + strlen (icomp) + 2;
1805
1806	if (sfile->filename != NULL)
1807	{
1808	sname = main_apphead_string (sfile->filename);
1809	scomp = (sfile->compressor == NULL) ? "" : sfile->compressor->ident;
1810	len += strlen (sname) + strlen (scomp) + 2;
1811	}
1812	else
1813	{
1814	sname = scomp = "";
1815	}
1816
1817	if ((appheader_used = main_malloc (len)) == NULL)
1818	{
1819	return ENOMEM;
1820	}
1821
1822	if (sfile->filename == NULL)
1823	{
1824	sprintf ((char*)appheader_used, "%s/%s", iname, icomp);
1825	}
1826	else
1827	{
1828	sprintf ((char*)appheader_used, "%s/%s/%s/%s", iname, icomp, sname, scomp);
1829	}
1830	}
1831
1832	xd3_set_appheader (stream, appheader_used, strlen ((char*)appheader_used));
1833
1834	return 0;
1835	}
1836	#endif
1837
1838	static void
1839	main_get_appheader_params (main_file file, char parsed, int output, const char type,
1840	main_file *other)
1841	{
1842	/* Set the filename if it was not specified. If output, option_stdout (-c) overrides. */
1843	if (file->filename == NULL && ! (output && option_stdout) && strcmp (parsed[0], "-") != 0)
1844	{
1845	file->filename = parsed[0];
1846
1847	if (other->filename != NULL) {
1848	/* Take directory from the other file, if it has one. */
1849	/* TODO: This results in nonsense names like /dev/foo.tar.gz
1850	* and probably the filename-default logic interferes with
1851	* multi-file operation and the standard file extension?
1852	* Possibly the name header is bad, should be off by default.
1853	* Possibly we just want to remember external/compression
1854	* settings. */
1855	char *last_slash = strrchr(other->filename, '/');
1856
1857	if (last_slash != NULL) {
1858	int dlen = last_slash - other->filename;
1859
1860	XD3_ASSERT(file->filename_copy == NULL);
1861	file->filename_copy = main_malloc(dlen + 2 + strlen(file->filename));
1862
1863	strncpy(file->filename_copy, other->filename, dlen);
1864	file->filename_copy[dlen] = '/';
1865	strcpy(file->filename_copy + dlen + 1, parsed[0]);
1866
1867	file->filename = file->filename_copy;
1868	}
1869	}
1870
1871	if (! option_quiet)
1872	{
1873	XPR(NT "using default %s filename: %s\n", type, file->filename);
1874	}
1875	}
1876
1877	/* Set the compressor, initiate de/recompression later. */
1878	if (file->compressor == NULL && *parsed[1] != 0)
1879	{
1880	file->compressor = main_get_compressor (parsed[1]);
1881	}
1882	}
1883
1884	static void
1885	main_get_appheader (xd3_stream stream, main_file ifile, main_file output, main_file sfile)
1886	{
1887	uint8_t *apphead;
1888	usize_t appheadsz;
1889	int ret;
1890
1891	/* The user may disable the application header. Once the appheader is set, this
1892	* disables setting it again. */
1893	if (! option_use_appheader) { return; }
1894
1895	ret = xd3_get_appheader (stream, & apphead, & appheadsz);
1896
1897	/* Ignore failure, it only means we haven't received a header yet. */
1898	if (ret != 0) { return; }
1899
1900	if (appheadsz > 0)
1901	{
1902	char start = (char)apphead;
1903	char *slash;
1904	int place = 0;
1905	char *parsed[4];
1906
1907	memset (parsed, 0, sizeof (parsed));
1908
1909	while ((slash = strchr (start, '/')) != NULL)
1910	{
1911	*slash = 0;
1912	parsed[place++] = start;
1913	start = slash + 1;
1914	}
1915
1916	parsed[place++] = start;
1917
1918	/* First take the output parameters. */
1919	if (place == 2 \|\| place == 4)
1920	{
1921	main_get_appheader_params (output, parsed, 1, "output", ifile);
1922	}
1923
1924	/* Then take the source parameters. */
1925	if (place == 4)
1926	{
1927	main_get_appheader_params (sfile, parsed+2, 0, "source", ifile);
1928	}
1929	}
1930
1931	option_use_appheader = 0;
1932	return;
1933	}
1934
1935	/******************************************************************************************
1936	Main I/O routines
1937	******************************************************************************************/
1938
1939	/* This function acts like the above except it may also try to recognize a compressed
1940	* input when the first buffer of data is read. The EXTERNAL_COMPRESSION code is called
1941	* to search for magic numbers. */
1942	static int
1943	main_read_primary_input (main_file *ifile,
1944	uint8_t *buf,
1945	usize_t size,
1946	usize_t *nread)
1947	{
1948	#if EXTERNAL_COMPRESSION
1949	if (option_decompress_inputs && ifile->flags & RD_FIRST)
1950	{
1951	ifile->flags &= ~RD_FIRST;
1952
1953	return main_decompress_input_check (ifile, buf, size, nread);
1954	}
1955	#endif
1956
1957	return main_file_read (ifile, buf, size, nread, "input read failed");
1958	}
1959
1960	/* This function simply writes the stream output buffer, if there is any. This is used
1961	* for both encode and decode commands. (The VCDIFF tools use main_print_func()). */
1962	static int
1963	main_write_output (xd3_stream* stream, main_file *ofile)
1964	{
1965	int ret;
1966
1967	if (stream->avail_out > 0 && (ret = main_file_write (ofile, stream->next_out, stream->avail_out, "write failed")))
1968	{
1969	return ret;
1970	}
1971
1972	return 0;
1973	}
1974
1975	/* Open the main output file, sets a default file name, initiate recompression. This
1976	* function is expected to fprint any error messages. */
1977	static int
1978	main_open_output (xd3_stream stream, main_file ofile)
1979	{
1980	int ret;
1981
1982	if (ofile->filename == NULL)
1983	{
1984	XSTDOUT_XF (ofile);
1985
1986	if (option_verbose > 1) { XPR(NT "using standard output: %s\n", ofile->filename); }
1987	}
1988	else
1989	{
1990	/* Stat the file to check for overwrite. */
1991	if (option_force == 0 && main_file_exists (ofile))
1992	{
1993	XPR(NT "to overwrite output file specify -f: %s\n", ofile->filename);
1994	return EEXIST;
1995	}
1996
1997	if ((ret = main_file_open (ofile, ofile->filename, XO_WRITE)))
1998	{
1999	return ret;
2000	}
2001
2002	if (option_verbose > 1) { XPR(NT "output file: %s\n", ofile->filename); }
2003	}
2004
2005	#if EXTERNAL_COMPRESSION
2006	/* Do output recompression. */
2007	if (ofile->compressor != NULL && option_recompress_outputs == 1)
2008	{
2009	if (! option_quiet)
2010	{
2011	XPR(NT "%s %s \| %s\n",
2012	ofile->compressor->recomp_cmdname,
2013	ofile->compressor->recomp_options,
2014	ofile->filename);
2015	}
2016
2017	if ((ret = main_recompress_output (ofile)))
2018	{
2019	return ret;
2020	}
2021	}
2022	#endif
2023
2024	return 0;
2025	}
2026
2027	/* This is called at different times for encoding and decoding. The encoder calls it
2028	* immediately, the decoder delays until the application header is received.
2029	* Stream may be NULL, in which case xd3_set_source is not called. */
2030	static int
2031	main_set_source (xd3_stream stream, int cmd, main_file sfile, xd3_source *source)
2032	{
2033	int ret = 0, i;
2034	uint8_t *tmp_buf = NULL;
2035
2036	/* Open it, check for seekability, set required xd3_source fields. */
2037	if (allow_fake_source)
2038	{
2039	sfile->mode = XO_READ;
2040	sfile->realname = sfile->filename;
2041	sfile->nread = 0;
2042	source->size = UINT64_MAX;
2043	}
2044	else
2045	{
2046	if ((ret = main_file_open (sfile, sfile->filename, XO_READ)) \|\|
2047	(ret = main_file_stat (sfile, & source->size, 1)))
2048	{
2049	goto error;
2050	}
2051	}
2052
2053	source->name = sfile->filename;
2054	source->ioh = sfile;
2055	source->curblkno = (xoff_t) -1;
2056	source->curblk = NULL;
2057
2058	#if EXTERNAL_COMPRESSION
2059	if (option_decompress_inputs)
2060	{
2061	/* If encoding, read the header to check for decompression. */
2062	if (IS_ENCODE (cmd))
2063	{
2064	usize_t nread;
2065	tmp_buf = main_malloc(XD3_ALLOCSIZE);
2066
2067	if ((ret = main_file_read (sfile, tmp_buf, XD3_ALLOCSIZE,
2068	& nread, "source read failed")))
2069	{
2070	goto error;
2071	}
2072
2073	/* Check known magic numbers. */
2074	for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)
2075	{
2076	const main_extcomp *decomp = & extcomp_types[i];
2077
2078	if ((nread > decomp->magic_size) &&
2079	memcmp (tmp_buf, decomp->magic, decomp->magic_size) == 0)
2080	{
2081	sfile->compressor = decomp;
2082	break;
2083	}
2084	}
2085
2086	if (sfile->compressor == NULL)
2087	{
2088	if (option_verbose > 2)
2089	{
2090	XPR(NT "source block 0 read (not compressed)\n");
2091	}
2092	}
2093	}
2094
2095	/* In either the encoder or decoder, start decompression. */
2096	if (sfile->compressor)
2097	{
2098	xoff_t osize = source->size;
2099
2100	if (osize > XD3_NODECOMPRESSSIZE)
2101	{
2102	XPR(NT "source file too large for external decompression: %s: %"Q"u\n",
2103	sfile->filename, osize);
2104	ret = XD3_INTERNAL;
2105	goto error;
2106	}
2107
2108	if ((ret = main_decompress_source (sfile, source)))
2109	{
2110	goto error;
2111	}
2112
2113	if (! option_quiet)
2114	{
2115	char s1[32], s2[32];
2116	XPR(NT "%s \| %s %s => %s %.1f%% [ %s , %s ]\n",
2117	sfile->filename,
2118	sfile->compressor->decomp_cmdname,
2119	sfile->compressor->decomp_options,
2120	sfile->realname,
2121	100.0 * source->size / osize,
2122	main_format_bcnt (osize, s1),
2123	main_format_bcnt (source->size, s2));
2124	}
2125	}
2126	}
2127	#endif
2128
2129	/* At this point we know source->size.
2130	* Source buffer, blksize, LRU init. */
2131	if (source->size < option_srcwinsz)
2132	{
2133	/* Reduce sizes to actual source size, read whole file */
2134	option_srcwinsz = source->size;
2135	source->blksize = source->size;
2136	lru_size = 1;
2137	}
2138	else
2139	{
2140	option_srcwinsz = max(option_srcwinsz, XD3_MINSRCWINSZ);
2141
2142	source->blksize = (option_srcwinsz / LRU_SIZE);
2143	lru_size = LRU_SIZE;
2144	}
2145
2146	main_blklru_list_init (& lru_list);
2147	main_blklru_list_init (& lru_free);
2148
2149	if (option_verbose)
2150	{
2151	static char buf[32];
2152
2153	XPR(NT "source %s winsize %s size %"Q"u\n",
2154	sfile->filename, main_format_bcnt(option_srcwinsz, buf), source->size);
2155	}
2156
2157	if (option_verbose > 1)
2158	{
2159	XPR(NT "source block size: %u\n", source->blksize);
2160	}
2161
2162	if ((lru = main_malloc (sizeof (main_blklru) * lru_size)) == NULL)
2163	{
2164	ret = ENOMEM;
2165	goto error;
2166	}
2167
2168	for (i = 0; i < lru_size; i += 1)
2169	{
2170	lru[i].blkno = (xoff_t) -1;
2171
2172	if ((lru[i].blk = main_malloc (source->blksize)) == NULL)
2173	{
2174	ret = ENOMEM;
2175	goto error;
2176	}
2177
2178	main_blklru_list_push_back (& lru_free, & lru[i]);
2179	}
2180
2181	if (stream && (ret = xd3_set_source (stream, source)))
2182	{
2183	XPR(NT XD3_LIB_ERRMSG (stream, ret));
2184	goto error;
2185	}
2186
2187	error:
2188	if (tmp_buf != NULL)
2189	{
2190	main_free (tmp_buf);
2191	}
2192
2193	return ret;
2194	}
2195
2196	static void
2197	main_set_winsize (main_file *ifile) {
2198	xoff_t file_size;
2199
2200	if (main_file_stat (ifile, &file_size, 0) == 0)
2201	{
2202	option_winsize = (usize_t) min(file_size, (xoff_t) option_winsize);
2203	}
2204
2205	option_winsize = max(option_winsize, XD3_ALLOCSIZE);
2206
2207	if (option_verbose > 1)
2208	{
2209	XPR(NT "input window size: %u\n", option_winsize);
2210	}
2211	}
2212
2213	/******************************************************************************************
2214	Source routines
2215	******************************************************************************************/
2216
2217	/* This is the callback for reading a block of source. This function is blocking and it
2218	* implements a small LRU.
2219	*
2220	* Note that it is possible for main_input() to handle getblk requests in a non-blocking
2221	* manner. If the callback is NULL then the caller of xd3_*_input() must handle the
2222	* XD3_GETSRCBLK return value and fill the source in the same way. See xd3_getblk for
2223	* details. To see an example of non-blocking getblk, see xdelta-test.h. */
2224	static int
2225	main_getblk_func (xd3_stream *stream,
2226	xd3_source *source,
2227	xoff_t blkno)
2228	{
2229	xoff_t pos = blkno * source->blksize;
2230	main_file sfile = (main_file) source->ioh;
2231	main_blklru *blru = NULL;
2232	usize_t onblk = xd3_bytes_on_srcblk (source, blkno);
2233	usize_t nread;
2234	int ret;
2235	int i;
2236
2237	if (allow_fake_source)
2238	{
2239	source->curblkno = blkno;
2240	source->onblk = onblk;
2241	source->curblk = lru[0].blk;
2242	return 0;
2243	}
2244
2245	if (do_not_lru)
2246	{
2247	/* Direct lookup assumes sequential scan w/o skipping blocks. */
2248	int idx = blkno % lru_size;
2249	if (lru[idx].blkno == blkno)
2250	{
2251	source->curblkno = blkno;
2252	source->onblk = onblk;
2253	source->curblk = lru[idx].blk;
2254	lru_hits += 1;
2255	return 0;
2256	}
2257
2258	if (lru[idx].blkno != -1LL &&
2259	lru[idx].blkno != blkno - lru_size)
2260	{
2261	return XD3_TOOFARBACK;
2262	}
2263	}
2264	else
2265	{
2266	/* Sequential search through LRU. */
2267	for (i = 0; i < lru_size; i += 1)
2268	{
2269	if (lru[i].blkno == blkno)
2270	{
2271	main_blklru_list_remove (& lru[i]);
2272	main_blklru_list_push_back (& lru_list, & lru[i]);
2273
2274	source->curblkno = blkno;
2275	source->onblk = onblk;
2276	source->curblk = lru[i].blk;
2277	lru_hits += 1;
2278	return 0;
2279	}
2280	}
2281	}
2282
2283	if (! main_blklru_list_empty (& lru_free))
2284	{
2285	blru = main_blklru_list_pop_front (& lru_free);
2286	}
2287	else if (! main_blklru_list_empty (& lru_list))
2288	{
2289	if (do_not_lru) {
2290	blru = & lru[blkno % lru_size];
2291	main_blklru_list_remove(blru);
2292	} else {
2293	blru = main_blklru_list_pop_front (& lru_list);
2294	}
2295	lru_misses += 1;
2296	}
2297
2298	lru_filled += 1;
2299
2300	if ((ret = main_file_seek (sfile, pos)))
2301	{
2302	return ret;
2303	}
2304
2305	if ((ret = main_file_read (sfile, (uint8_t*) blru->blk, source->blksize,
2306	& nread, "source read failed")))
2307	{
2308	return ret;
2309	}
2310
2311	if (nread != onblk)
2312	{
2313	XPR(NT "source file size change: %s\n", sfile->filename);
2314	return XD3_INTERNAL;
2315	}
2316
2317	main_blklru_list_push_back (& lru_list, blru);
2318
2319	if (option_verbose > 3)
2320	{
2321	if (blru->blkno != -1LL)
2322	{
2323	XPR(NT "source block %"Q"u ejects %"Q"u (lru_hits=%u, lru_misses=%u, lru_filled=%u)\n",
2324	blkno, blru->blkno, lru_hits, lru_misses, lru_filled);
2325	}
2326	else
2327	{
2328	XPR(NT "source block %"Q"u read (lru_hits=%u, lru_misses=%u, lru_filled=%u)\n",
2329	blkno, lru_hits, lru_misses, lru_filled);
2330	}
2331	}
2332
2333	blru->blkno = blkno;
2334	source->curblk = blru->blk;
2335	source->curblkno = blkno;
2336	source->onblk = onblk;
2337
2338	return 0;
2339	}
2340
2341	/******************************************************************************************
2342	Main routines
2343	******************************************************************************************/
2344
2345	/* This is a generic input function. It calls the xd3_encode_input or xd3_decode_input
2346	* functions and makes calls to the various input handling routines above, which
2347	* coordinate external decompression.
2348	*/
2349	static int
2350	main_input (xd3_cmd cmd,
2351	main_file *ifile,
2352	main_file *ofile,
2353	main_file *sfile)
2354	{
2355	int ret;
2356	xd3_stream stream;
2357	usize_t nread;
2358	int stream_flags = 0;
2359	xd3_config config;
2360	xd3_source source;
2361	xoff_t last_total_in = 0;
2362	xoff_t last_total_out = 0;
2363	long start_time;
2364	int stdout_only = 0;
2365
2366	int (input_func) (xd3_stream);
2367	int (output_func) (xd3_stream, main_file *);
2368
2369	memset (& source, 0, sizeof (source));
2370	memset (& config, 0, sizeof (config));
2371
2372	config.alloc = main_alloc;
2373	config.freef = main_free1;
2374	config.sec_data.ngroups = 1;
2375	config.sec_addr.ngroups = 1;
2376	config.sec_inst.ngroups = 1;
2377	config.iopt_size = option_iopt_size;
2378	config.sprevsz = option_sprevsz;
2379
2380	do_not_lru = 0;
2381
2382	start_time = get_millisecs_now ();
2383
2384	/* main_input setup. */
2385	switch ((int) cmd)
2386	{
2387	#if VCDIFF_TOOLS
2388	if (1) { case CMD_PRINTHDR: stream_flags = XD3_JUST_HDR; }
2389	else if (1) { case CMD_PRINTHDRS: stream_flags = XD3_SKIP_WINDOW; }
2390	else { case CMD_PRINTDELTA: stream_flags = XD3_SKIP_EMIT; }
2391	ifile->flags \|= RD_NONEXTERNAL;
2392	input_func = xd3_decode_input;
2393	output_func = main_print_func;
2394	stream_flags \|= XD3_ADLER32_NOVER;
2395	stdout_only = 1;
2396	break;
2397	#endif
2398	#if XD3_ENCODER
2399	case CMD_ENCODE:
2400	do_not_lru = 1;
2401	input_func = xd3_encode_input;
2402	output_func = main_write_output;
2403
2404	if (option_use_checksum) { stream_flags \|= XD3_ADLER32; }
2405	if (option_use_secondary)
2406	{
2407	/* The default secondary compressor is DJW, if it's compiled, being used, etc. */
2408	if (option_secondary == NULL)
2409	{
2410	if (SECONDARY_DJW) { stream_flags \|= XD3_SEC_DJW; }
2411	}
2412	else
2413	{
2414	if (strcmp (option_secondary, "fgk") == 0 && SECONDARY_FGK)
2415	{
2416	stream_flags \|= XD3_SEC_FGK;
2417	}
2418	else if (strcmp (option_secondary, "djw") == 0 && SECONDARY_DJW)
2419	{
2420	stream_flags \|= XD3_SEC_DJW;
2421	}
2422	else
2423	{
2424	XPR(NT "unrecognized secondary compressor type: %s\n", option_secondary);
2425	return EXIT_FAILURE;
2426	}
2427	}
2428	}
2429	if (option_no_compress) { stream_flags \|= XD3_NOCOMPRESS; }
2430	if (option_use_altcodetable) { stream_flags \|= XD3_ALT_CODE_TABLE; }
2431	if (option_smatch_config)
2432	{
2433	char s = option_smatch_config, e;
2434	int values[XD3_SOFTCFG_VARCNT];
2435	int got;
2436
2437	config.smatch_cfg = XD3_SMATCH_SOFT;
2438
2439	for (got = 0; got < XD3_SOFTCFG_VARCNT; got += 1, s = e + 1)
2440	{
2441	values[got] = strtol (s, &e, 10);
2442
2443	if ((values[got] < 0) \|\|
2444	(e == s) \|\|
2445	(got < XD3_SOFTCFG_VARCNT-1 && *e == 0) \|\|
2446	(got == XD3_SOFTCFG_VARCNT-1 && *e != 0))
2447	{
2448	XPR(NT "invalid string match specifier (-C) %d: %s\n",
2449	got, s);
2450	return EXIT_FAILURE;
2451	}
2452	}
2453
2454	config.smatcher_soft.large_look = values[0];
2455	config.smatcher_soft.large_step = values[1];
2456	config.smatcher_soft.small_look = values[2];
2457	config.smatcher_soft.small_chain = values[3];
2458	config.smatcher_soft.small_lchain = values[4];
2459	config.smatcher_soft.max_lazy = values[5];
2460	config.smatcher_soft.long_enough = values[6];
2461	}
2462	else
2463	{
2464	if (option_verbose > 1)
2465	{
2466	XPR(NT "compression level: %d\n", option_level);
2467	}
2468	if (option_level == 0)
2469	{
2470	stream_flags \|= XD3_NOCOMPRESS;
2471	config.smatch_cfg = XD3_SMATCH_FASTEST;
2472	}
2473	else if (option_level == 1) { config.smatch_cfg = XD3_SMATCH_FASTEST; }
2474	else if (option_level <= 5) { config.smatch_cfg = XD3_SMATCH_FAST; }
2475	else if (option_level == 6) { config.smatch_cfg = XD3_SMATCH_DEFAULT; }
2476	else { config.smatch_cfg = XD3_SMATCH_SLOW; }
2477	}
2478	break;
2479	#endif
2480	case CMD_DECODE:
2481	if (option_use_checksum == 0) { stream_flags \|= XD3_ADLER32_NOVER; }
2482	stream_flags = 0;
2483	ifile->flags \|= RD_NONEXTERNAL;
2484	input_func = xd3_decode_input;
2485	output_func = main_write_output;
2486	break;
2487	default:
2488	XPR(NT "internal error\n");
2489	return EXIT_FAILURE;
2490	}
2491
2492	main_set_winsize (ifile);
2493
2494	if ((main_bdata = main_malloc (option_winsize)) == NULL)
2495	{
2496	return EXIT_FAILURE;
2497	}
2498
2499	if (IS_ENCODE (cmd))
2500	{
2501	/* When encoding, open the source file, possibly decompress it. The decoder delays
2502	* this step until XD3_GOTHEADER. */
2503	if (sfile->filename != NULL && (ret = main_set_source (NULL, cmd, sfile, & source)))
2504	{
2505	return EXIT_FAILURE;
2506	}
2507	}
2508
2509	config.winsize = option_winsize;
2510	config.srcwin_maxsz = option_srcwinsz;
2511	config.getblk = main_getblk_func;
2512	config.flags = stream_flags;
2513
2514	if ((ret = xd3_config_stream (& stream, & config)))
2515	{
2516	XPR(NT XD3_LIB_ERRMSG (& stream, ret));
2517	return EXIT_FAILURE;
2518	}
2519
2520	if (IS_ENCODE (cmd) && sfile->filename != NULL &&
2521	(ret = xd3_set_source (& stream, & source)))
2522	{
2523	XPR(NT XD3_LIB_ERRMSG (& stream, ret));
2524	return EXIT_FAILURE;
2525	}
2526
2527	/* This times each window. */
2528	get_millisecs_since ();
2529
2530	/* Main input loop. */
2531	do
2532	{
2533	xoff_t input_offset;
2534	xoff_t input_remain;
2535	usize_t try_read;
2536
2537	input_offset = ifile->nread;
2538
2539	input_remain = XOFF_T_MAX - input_offset;
2540
2541	try_read = (usize_t) min ((xoff_t) config.winsize, input_remain);
2542
2543	if ((ret = main_read_primary_input (ifile, main_bdata, try_read, & nread)))
2544	{
2545	return EXIT_FAILURE;
2546	}
2547
2548	/* If we've reached EOF tell the stream to flush. */
2549	if (nread < try_read)
2550	{
2551	stream_flags \|= XD3_FLUSH;
2552	xd3_set_flags (& stream, stream_flags);
2553	}
2554
2555	#if XD3_ENCODER
2556	/* After the first main_read_primary_input completes, we know all the information
2557	* needed to encode the application header. */
2558	if (cmd == CMD_ENCODE && (ret = main_set_appheader (& stream, ifile, sfile)))
2559	{
2560	return EXIT_FAILURE;
2561	}
2562	#endif
2563	xd3_avail_input (& stream, main_bdata, nread);
2564
2565	/* If we read zero bytes after encoding at least one window... */
2566	if (nread == 0 && stream.current_window > 0) {
2567	break;
2568	}
2569
2570	again:
2571	ret = input_func (& stream);
2572
2573	switch (ret)
2574	{
2575	case XD3_INPUT:
2576	continue;
2577
2578	case XD3_GOTHEADER:
2579	{
2580	XD3_ASSERT (stream.current_window == 0);
2581
2582	/* Need to process the appheader as soon as possible. It may contain a
2583	* suggested default filename/decompression routine for the ofile, and it may
2584	* contain default/decompression routine for the sources. */
2585	if (cmd == CMD_DECODE)
2586	{
2587	int have_src = sfile->filename != NULL;
2588	int need_src = xd3_decoder_needs_source (& stream);
2589	int recv_src;
2590
2591	/* May need to set the sfile->filename if none was given. */
2592	main_get_appheader (& stream, ifile, ofile, sfile);
2593
2594	recv_src = sfile->filename != NULL;
2595
2596	/* Check if the user expected a source to be required although it was not. */
2597	if (have_src && ! need_src && ! option_quiet)
2598	{
2599	XPR(NT "warning: output window %"Q"u does not copy source\n", stream.current_window);
2600	}
2601
2602	/* Check if we have no source name and need one. */
2603	/* TODO: this doesn't fire due to cpyblocks_ calculation check? */
2604	if (need_src && ! recv_src)
2605	{
2606	XPR(NT "input requires a source file, use -s\n");
2607	return EXIT_FAILURE;
2608	}
2609
2610	/* Now open the source file. */
2611	if (need_src && (ret = main_set_source (& stream, cmd, sfile, & source)))
2612	{
2613	return EXIT_FAILURE;
2614	}
2615	}
2616	else if (cmd == CMD_PRINTHDR \|\|
2617	cmd == CMD_PRINTHDRS \|\|
2618	cmd == CMD_PRINTDELTA)
2619	{
2620	if (xd3_decoder_needs_source (& stream) && sfile->filename == NULL)
2621	{
2622	allow_fake_source = 1;
2623	sfile->filename = "<placeholder>";
2624	main_set_source (& stream, cmd, sfile, & source);
2625	}
2626	}
2627	}
2628	/* FALLTHROUGH */
2629	case XD3_WINSTART:
2630	{
2631	/* e.g., set or unset XD3_SKIP_WINDOW. */
2632	/* xd3_set_flags (& stream, stream_flags); */
2633	goto again;
2634	}
2635
2636	case XD3_OUTPUT:
2637	{
2638	if (option_no_output == 0)
2639	{
2640	/* Defer opening the output file until the stream produces its first
2641	* output for both encoder and decoder, this way we delay long enough for
2642	* the decoder to receive the application header. (Or longer if there are
2643	* skipped windows, but I can't think of any reason not to delay open.) */
2644
2645	if (! main_file_isopen (ofile) && (ret = main_open_output (& stream, ofile)) != 0)
2646	{
2647	return EXIT_FAILURE;
2648	}
2649	if ((ret = output_func (& stream, ofile)) && (ret != PRINTHDR_SPECIAL))
2650	{
2651	return EXIT_FAILURE;
2652	}
2653	if (ret == PRINTHDR_SPECIAL)
2654	{
2655	xd3_abort_stream (& stream);
2656	ret = EXIT_SUCCESS;
2657	goto done;
2658	}
2659	ret = 0;
2660	}
2661
2662	xd3_consume_output (& stream);
2663	goto again;
2664	}
2665
2666	case XD3_WINFINISH:
2667	{
2668	if (IS_ENCODE (cmd) \|\| cmd == CMD_DECODE)
2669	{
2670	if (! option_quiet && IS_ENCODE (cmd) && main_file_isopen (sfile))
2671	{
2672	/* Warn when no source copies are found */
2673	if (! xd3_encoder_used_source (& stream))
2674	{
2675	XPR(NT "warning: input window %"Q"u..%"Q"u has no source copies\n",
2676	stream.current_window * option_winsize,
2677	(stream.current_window+1) * option_winsize);
2678	}
2679
2680	/* Limited instruction buffer size affects source copies */
2681	if (option_verbose > 1 && stream.i_slots_used > stream.iopt_size)
2682	{
2683	XPR(NT "warning: input position %"Q"u overflowed instruction buffer, "
2684	"needed %u (vs. %u), consider raising -I\n",
2685	stream.current_window * option_winsize,
2686	stream.i_slots_used, stream.iopt_size);
2687	}
2688	}
2689
2690	if (option_verbose)
2691	{
2692	char rrateavg[32], wrateavg[32], tm[32];
2693	char rdb[32], wdb[32];
2694	char trdb[32], twdb[32];
2695	long millis = get_millisecs_since ();
2696	usize_t this_read = (usize_t)(stream.total_in - last_total_in);
2697	usize_t this_write = (usize_t)(stream.total_out - last_total_out);
2698	last_total_in = stream.total_in;
2699	last_total_out = stream.total_out;
2700
2701	if (option_verbose > 1)
2702	{
2703	XPR(NT "%"Q"u: in %s (%s): out %s (%s): total in %s: out %s: %s\n",
2704	stream.current_window,
2705	main_format_bcnt (this_read, rdb),
2706	main_format_rate (this_read, millis, rrateavg),
2707	main_format_bcnt (this_write, wdb),
2708	main_format_rate (this_write, millis, wrateavg),
2709	main_format_bcnt (stream.total_in, trdb),
2710	main_format_bcnt (stream.total_out, twdb),
2711	main_format_millis (millis, tm));
2712	}
2713	else
2714	{
2715	XPR(NT "%"Q"u: in %s: out %s: total in %s: out %s: %s\n",
2716	stream.current_window,
2717	main_format_bcnt (this_read, rdb),
2718	main_format_bcnt (this_write, wdb),
2719	main_format_bcnt (stream.total_in, trdb),
2720	main_format_bcnt (stream.total_out, twdb),
2721	main_format_millis (millis, tm));
2722	}
2723	}
2724	}
2725	goto again;
2726	}
2727
2728	default:
2729	/* input_func() error */
2730	XPR(NT XD3_LIB_ERRMSG (& stream, ret));
2731	return EXIT_FAILURE;
2732	}
2733	}
2734	while (nread == config.winsize);
2735	done:
2736	/* Close the inputs. (ifile must be open, sfile may be open) */
2737	main_file_close (ifile);
2738	main_file_close (sfile);
2739
2740	/* If output file is not open yet because of delayed-open, it means we never encountered
2741	* a window in the delta, but it could have had a VCDIFF header? TODO: solve this
2742	* elsewhere. For now, it prints "nothing to output" below, but the check doesn't
2743	* happen in case of option_no_output. */
2744	if (! option_no_output)
2745	{
2746	if (!stdout_only && ! main_file_isopen (ofile))
2747	{
2748	XPR(NT "nothing to output: %s\n", ifile->filename);
2749	return EXIT_FAILURE;
2750	}
2751
2752	/* Have to close the output before calling main_external_compression_finish, or else it hangs. */
2753	if (main_file_close (ofile) != 0)
2754	{
2755	return EXIT_FAILURE;
2756	}
2757	}
2758
2759	#if EXTERNAL_COMPRESSION
2760	if ((ret = main_external_compression_finish ()))
2761	{
2762	XPR(NT "external compression commands failed\n");
2763	return EXIT_FAILURE;
2764	}
2765	#endif
2766
2767	if ((ret = xd3_close_stream (& stream)))
2768	{
2769	XPR(NT XD3_LIB_ERRMSG (& stream, ret));
2770	return EXIT_FAILURE;
2771	}
2772
2773	if (option_verbose > 1)
2774	{
2775	XPR(NT "scanner configuration: %s\n", stream.smatcher.name);
2776	XPR(NT "target hash table size: %u\n", stream.small_hash.size);
2777	if (sfile->filename != NULL)
2778	{
2779	XPR(NT "source hash table size: %u\n", stream.large_hash.size);
2780	}
2781	}
2782
2783	if (option_verbose > 2)
2784	{
2785	XPR(NT "source copies: %"Q"u (%"Q"u bytes)\n", stream.n_scpy, stream.l_scpy);
2786	XPR(NT "target copies: %"Q"u (%"Q"u bytes)\n", stream.n_tcpy, stream.l_tcpy);
2787	XPR(NT "adds: %"Q"u (%"Q"u bytes)\n", stream.n_add, stream.l_add);
2788	XPR(NT "runs: %"Q"u (%"Q"u bytes)\n", stream.n_run, stream.l_run);
2789	}
2790
2791	xd3_free_stream (& stream);
2792
2793	if (option_verbose)
2794	{
2795	char tm[32];
2796	long end_time = get_millisecs_now ();
2797	XPR(NT "finished in %s; input %"Q"u output %"Q"u bytes (%0.2f%%)\n",
2798	main_format_millis (end_time - start_time, tm),
2799	ifile->nread, ofile->nwrite,
2800	100.0 * ofile->nwrite / ifile->nread);
2801	}
2802
2803	return EXIT_SUCCESS;
2804	}
2805
2806	/* free memory before exit, reset single-use variables. */
2807	static void
2808	main_cleanup (void)
2809	{
2810	int i;
2811
2812	if (appheader_used != NULL &&
2813	appheader_used != option_appheader)
2814	{
2815	main_free (appheader_used);
2816	appheader_used = NULL;
2817	}
2818
2819	main_free (main_bdata);
2820	main_bdata = NULL;
2821
2822	#if EXTERNAL_COMPRESSION
2823	main_free (ext_tmpfile);
2824	ext_tmpfile = NULL;
2825	#endif
2826
2827	for (i = 0; lru && i < lru_size; i += 1)
2828	{
2829	main_free (lru[i].blk);
2830	}
2831
2832	main_free (lru);
2833	lru = NULL;
2834
2835	lru_hits = 0;
2836	lru_misses = 0;
2837	lru_filled = 0;
2838
2839	XD3_ASSERT (main_mallocs == 0);
2840	}
2841
2842	static void
2843	setup_environment (int argc,
2844	char **argv,
2845	int *argc_out,
2846	char ***argv_out,
2847	char ***argv_free,
2848	char **env_free)
2849	{
2850	int n, i, i0;
2851	char p, v = getenv("XDELTA");
2852	if (v == NULL) {
2853	(*argc_out) = argc;
2854	(*argv_out) = argv;
2855	(*argv_free) = NULL;
2856	(*env_free) = NULL;
2857	return;
2858	}
2859
2860	(*env_free) = main_malloc(strlen(v) + 1);
2861	strcpy(*env_free, v);
2862
2863	/* Space needed for extra args, at least # of spaces */
2864	n = argc + 1;
2865	for (p = env_free; p != 0; ) {
2866	if (*p++ == ' ') {
2867	n++;
2868	}
2869	}
2870
2871	(argv_free) = main_malloc(sizeof(char) * (n + 1));
2872	(argv_out) = (argv_free);
2873	(*argv_out)[0] = argv[0];
2874	(*argv_out)[n] = NULL;
2875
2876	i = 1;
2877	for (p = env_free; p != 0; ) {
2878	(*argv_out)[i++] = p;
2879	while (p != ' ' && p != 0) {
2880	p++;
2881	}
2882	while (*p == ' ') {
2883	*p++ = 0;
2884	}
2885	}
2886
2887	for (i0 = 1; i0 < argc; i0++) {
2888	(*argv_out)[i++] = argv[i0];
2889	}
2890
2891	/* Counting spaces is an upper bound, argv stays NULL terminated. */
2892	(*argc_out) = i;
2893	while (i <= n) {
2894	(*argv_out)[i++] = NULL;
2895	}
2896	}
2897
2898	int
2899	#if PYTHON_MODULE \|\| SWIG_MODULE
2900	xd3_main_cmdline (int argc, char **argv)
2901	#else
2902	main (int argc, char **argv)
2903	#endif
2904	{
2905	xd3_cmd cmd;
2906	main_file ifile;
2907	main_file ofile;
2908	main_file sfile;
2909	static const char *flags = "0123456789cdefhnqvDJNORTVs:B:C:E:F:I:L:O:M:P:W:A::S::";
2910	int my_optind;
2911	char *my_optarg;
2912	char *my_optstr;
2913	char *sfilename;
2914	int env_argc;
2915	char **env_argv;
2916	char *free_argv; / malloc() in setup_environment() */
2917	char free_value; / malloc() in setup_environment() */
2918	int ret;
2919
2920	#ifdef _WIN32
2921	GetStartupInfo(&winStartupInfo);
2922	setvbuf(stderr, NULL, _IONBF, 0); /* Do not buffer stderr */
2923	#endif
2924
2925	main_file_init (& ifile);
2926	main_file_init (& ofile);
2927	main_file_init (& sfile);
2928
2929	reset_defaults();
2930
2931	free_argv = NULL;
2932	free_value = NULL;
2933	setup_environment(argc, argv, &env_argc, &env_argv, &free_argv, &free_value);
2934	cmd = CMD_NONE;
2935	sfilename = NULL;
2936	my_optind = 1;
2937	argv = env_argv;
2938	argc = env_argc;
2939	program_name = env_argv[0];
2940	extcomp_types[0].recomp_cmdname = program_name;
2941	extcomp_types[0].decomp_cmdname = program_name;
2942	takearg:
2943	my_optarg = NULL;
2944	my_optstr = argv[my_optind];
2945	/* This doesn't use getopt() because it makes trouble for -P & python which reenter
2946	* main() and thus care about freeing all memory. I never had much trust for getopt
2947	* anyway, it's too opaque. This implements a fairly standard non-long-option getopt
2948	* with support for named operations (e.g., "xdelta3 [encode\|decode\|printhdr...] < in >
2949	* out"). */
2950	if (my_optstr)
2951	{
2952	if (*my_optstr == '-') { my_optstr += 1; }
2953	else if (cmd == CMD_NONE) { goto nonflag; }
2954	else { my_optstr = NULL; }
2955	}
2956	while (my_optstr)
2957	{
2958	char *s;
2959	my_optarg = NULL;
2960	if ((ret = *my_optstr++) == 0) { my_optind += 1; goto takearg; }
2961
2962	/* Option handling: first check for one ':' following the option in flags, then
2963	* check for two. The syntax allows:
2964	*
2965	* 1. -Afoo defines optarg="foo"
2966	* 2. -A foo defines optarg="foo"
2967	* 3. -A "" defines optarg="" (allows optional empty-string)
2968	* 4. -A [EOA or -moreargs] error (mandatory case)
2969	* 5. -A [EOA -moreargs] defines optarg=NULL (optional case)
2970	* 6. -A=foo defines optarg="foo"
2971	* 7. -A= defines optarg="" (mandatory case)
2972	* 8. -A= defines optarg=NULL (optional case)
2973	*
2974	* See tests in test_command_line_arguments().
2975	*/
2976	s = strchr (flags, ret);
2977	if (s && s[1] && s[1] == ':')
2978	{
2979	int eqcase = 0;
2980	int option = s[2] && s[2] == ':';
2981
2982	/* Case 1, set optarg to the remaining characters. */
2983	my_optarg = my_optstr;
2984	my_optstr = "";
2985
2986	/* Case 2-5 */
2987	if (*my_optarg == 0)
2988	{
2989	/* Condition 4-5 */
2990	int have_arg = my_optind < (argc - 1) && *argv[my_optind+1] != '-';
2991
2992	if (! have_arg)
2993	{
2994	if (! option)
2995	{
2996	/* Case 4 */
2997	XPR(NT "-%c: requires an argument\n", ret);
2998	ret = EXIT_FAILURE;
2999	goto cleanup;
3000	}
3001	/* Case 5. */
3002	my_optarg = NULL;
3003	}
3004	else
3005	{
3006	/* Case 2-3. */
3007	my_optarg = argv[++my_optind];
3008	}
3009	}
3010	/* Case 6-8. */
3011	else if (*my_optarg == '=')
3012	{
3013	/* Remove the = in all cases. */
3014	my_optarg += 1;
3015	eqcase = 1;
3016
3017	if (option && *my_optarg == 0)
3018	{
3019	/* Case 8. */
3020	my_optarg = NULL;
3021	}
3022	}
3023	}
3024
3025	switch (ret)
3026	{
3027	/* case: if no '-' was found, maybe check for a command name. */
3028	nonflag:
3029	if (strcmp (my_optstr, "decode") == 0) { cmd = CMD_DECODE; }
3030	else if (strcmp (my_optstr, "encode") == 0)
3031	{
3032	#if XD3_ENCODER
3033	cmd = CMD_ENCODE;
3034	#else
3035	XPR(NT "encoder support not compiled\n");
3036	return EXIT_FAILURE;
3037	#endif
3038	}
3039	else if (strcmp (my_optstr, "config") == 0) { cmd = CMD_CONFIG; }
3040	#if REGRESSION_TEST
3041	else if (strcmp (my_optstr, "test") == 0) { cmd = CMD_TEST; }
3042	#endif
3043	#if VCDIFF_TOOLS
3044	else if (strcmp (my_optstr, "printhdr") == 0) { cmd = CMD_PRINTHDR; }
3045	else if (strcmp (my_optstr, "printhdrs") == 0) { cmd = CMD_PRINTHDRS; }
3046	else if (strcmp (my_optstr, "printdelta") == 0) { cmd = CMD_PRINTDELTA; }
3047	#endif
3048
3049	/* If no option was found and still no command, let the default command be
3050	* encode. The remaining args are treated as filenames. */
3051	if (cmd == CMD_NONE)
3052	{
3053	cmd = CMD_DEFAULT;
3054	my_optstr = NULL;
3055	break;
3056	}
3057	else
3058	{
3059	/* But if we find a command name, continue the getopt loop. */
3060	my_optind += 1;
3061	goto takearg;
3062	}
3063
3064	/* gzip-like options */
3065	case '0': case '1': case '2': case '3': case '4':
3066	case '5': case '6': case '7': case '8': case '9':
3067	option_level = ret - '0';
3068	break;
3069	case 'f': option_force = 1; break;
3070	case 'v': option_verbose += 1; option_quiet = 0; break;
3071	case 'q': option_quiet = 1; option_verbose = 0; break;
3072	case 'c': option_stdout = 1; break;
3073	case 'd':
3074	if (cmd == CMD_NONE) { cmd = CMD_DECODE; }
3075	else { ret = main_help (); goto exit; }
3076	break;
3077	case 'e':
3078	#if XD3_ENCODER
3079	if (cmd == CMD_NONE) { cmd = CMD_ENCODE; }
3080	else { ret = main_help (); goto exit; }
3081	break;
3082	#else
3083	XPR(NT "encoder support not compiled\n");
3084	return EXIT_FAILURE;
3085	#endif
3086
3087	case 'n': option_use_checksum = 0; break;
3088	case 'N': option_no_compress = 1; break;
3089	case 'T': option_use_altcodetable = 1; break;
3090	case 'C': option_smatch_config = my_optarg; break;
3091	case 'J': option_no_output = 1; break;
3092	case 'S': if (my_optarg == NULL) { option_use_secondary = 0; }
3093	else { option_use_secondary = 1; option_secondary = my_optarg; } break;
3094	case 'A': if (my_optarg == NULL) { option_use_appheader = 0; }
3095	else { option_appheader = (uint8_t*) my_optarg; } break;
3096	case 'B':
3097	if ((ret = main_atou (my_optarg, & option_srcwinsz, XD3_MINSRCWINSZ,
3098	0, 'B')))
3099	{
3100	goto exit;
3101	}
3102	break;
3103	case 'I':
3104	if ((ret = main_atou (my_optarg, & option_iopt_size, 0,
3105	0, 'I')))
3106	{
3107	goto exit;
3108	}
3109	break;
3110	case 'P':
3111	if ((ret = main_atou (my_optarg, & option_sprevsz, 0,
3112	0, 'P')))
3113	{
3114	goto exit;
3115	}
3116	break;
3117	case 'W':
3118	if ((ret = main_atou (my_optarg, & option_winsize, XD3_ALLOCSIZE,
3119	XD3_HARDMAXWINSIZE, 'W')))
3120	{
3121	goto exit;
3122	}
3123	break;
3124	case 'D':
3125	#if EXTERNAL_COMPRESSION == 0
3126	if (option_verbose > 0)
3127	{
3128	XPR(NT "warning: -D option ignored, "
3129	"external compression support was not compiled\n");
3130	}
3131	#else
3132	option_decompress_inputs = 0;
3133	#endif
3134	break;
3135	case 'R':
3136	#if EXTERNAL_COMPRESSION == 0
3137	if (option_verbose > 0)
3138	{
3139	XPR(NT "warning: -R option ignored, "
3140	"external compression support was not compiled\n");
3141	}
3142	#else
3143	option_recompress_outputs = 0;
3144	#endif
3145	break;
3146	case 's':
3147	if (sfilename != NULL)
3148	{
3149	XPR(NT "specify only one source file\n");
3150	goto cleanup;
3151	}
3152
3153	sfilename = my_optarg;
3154	break;
3155
3156	case 'V':
3157	ret = main_version (); goto exit;
3158	default:
3159	ret = main_help (); goto exit;
3160	}
3161	}
3162
3163	option_source_filename = sfilename;
3164
3165	/* In case there were no arguments, set the default command. */
3166	if (cmd == CMD_NONE) { cmd = CMD_DEFAULT; }
3167
3168	argc -= my_optind;
3169	argv += my_optind;
3170
3171	/* There may be up to two more arguments. */
3172	if (argc > 2)
3173	{
3174	XPR(NT "too many filenames: %s ...\n", argv[2]);
3175	ret = EXIT_FAILURE;
3176	goto cleanup;
3177	}
3178
3179	ifile.flags = RD_FIRST;
3180	sfile.flags = RD_FIRST;
3181	sfile.filename = option_source_filename;
3182
3183	/* The infile takes the next argument, if there is one. But if not, infile is set to
3184	* stdin. */
3185	if (argc > 0)
3186	{
3187	ifile.filename = argv[0];
3188
3189	if ((ret = main_file_open (& ifile, ifile.filename, XO_READ)))
3190	{
3191	goto cleanup;
3192	}
3193	}
3194	else
3195	{
3196	XSTDIN_XF (& ifile);
3197	}
3198
3199	/* The ofile takes the following argument, if there is one. But if not, it is left NULL
3200	* until the application header is processed. It will be set in main_open_output. */
3201	if (argc > 1)
3202	{
3203	/* Check for conflicting arguments. */
3204	if (option_stdout && ! option_quiet)
3205	{
3206	XPR(NT "warning: -c option overrides output filename: %s\n", argv[1]);
3207	}
3208
3209	if (! option_stdout) { ofile.filename = argv[1]; }
3210	}
3211
3212	switch (cmd)
3213	{
3214	case CMD_PRINTHDR:
3215	case CMD_PRINTHDRS:
3216	case CMD_PRINTDELTA:
3217	#if XD3_ENCODER
3218	case CMD_ENCODE:
3219	#endif
3220	case CMD_DECODE:
3221	ret = main_input (cmd, & ifile, & ofile, & sfile);
3222	break;
3223
3224	#if REGRESSION_TEST
3225	case CMD_TEST:
3226	ret = xd3_selftest ();
3227	break;
3228	#endif
3229
3230	case CMD_CONFIG:
3231	ret = main_config ();
3232	break;
3233
3234	default:
3235	ret = main_help ();
3236	break;
3237	}
3238
3239	#if EXTERNAL_COMPRESSION
3240	if (ext_tmpfile != NULL)
3241	{
3242	unlink (ext_tmpfile);
3243	}
3244	#endif
3245
3246	if (0)
3247	{
3248	cleanup:
3249	ret = EXIT_FAILURE;
3250	exit:
3251	(void)0;
3252	}
3253
3254	main_file_cleanup (& ifile);
3255	main_file_cleanup (& ofile);
3256	main_file_cleanup (& sfile);
3257
3258	main_free (free_argv);
3259	main_free (free_value);
3260
3261	main_cleanup ();
3262
3263	fflush (stdout);
3264	fflush (stderr);
3265	return ret;
3266	}
3267
3268	static int
3269	main_help (void)
3270	{
3271	/* Note: update wiki when command-line features change */
3272	main_version ();
3273	DP(RINT "usage: xdelta3 [command/options] [input [output]]\n");
3274	DP(RINT "special command names:\n");
3275	DP(RINT " config prints xdelta3 configuration\n");
3276	DP(RINT " decode decompress the input\n");
3277	DP(RINT " encode compress the input%s\n", XD3_ENCODER ? "" : " [Not compiled]");
3278	#if REGRESSION_TEST
3279	DP(RINT " test run the builtin tests\n");
3280	#endif
3281	#if VCDIFF_TOOLS
3282	DP(RINT "special commands for VCDIFF inputs:\n");
3283	DP(RINT " printdelta print information about the entire delta\n");
3284	DP(RINT " printhdr print information about the first window\n");
3285	DP(RINT " printhdrs print information about all windows\n");
3286	#endif
3287	DP(RINT "standard options:\n");
3288	DP(RINT " -0 .. -9 compression level\n");
3289	DP(RINT " -c use stdout\n");
3290	DP(RINT " -d decompress\n");
3291	DP(RINT " -e compress%s\n", XD3_ENCODER ? "" : " [Not compiled]");
3292	DP(RINT " -f force overwrite\n");
3293	DP(RINT " -h show help\n");
3294	DP(RINT " -q be quiet\n");
3295	DP(RINT " -v be verbose (max 2)\n");
3296	DP(RINT " -V show version\n");
3297
3298	DP(RINT "memory options:\n");
3299	DP(RINT " -B bytes source window size\n");
3300	DP(RINT " -W bytes input window size\n");
3301	DP(RINT " -P size compression duplicates window\n");
3302	DP(RINT " -I size instruction buffer size (0 = unlimited)\n");
3303
3304	DP(RINT "compression options:\n");
3305	DP(RINT " -s source source file to copy from (if any)\n");
3306	DP(RINT " -S [djw\|fgk] enable/disable secondary compression\n");
3307	DP(RINT " -N disable small string-matching compression\n");
3308	DP(RINT " -D disable external decompression (encode/decode)\n");
3309	DP(RINT " -R disable external recompression (decode)\n");
3310	DP(RINT " -n disable checksum (encode/decode)\n");
3311	DP(RINT " -C soft config (encode, undocumented)\n");
3312	DP(RINT " -A [apphead] disable/provide application header (encode)\n");
3313
3314	#if XD3_DEBUG > 0
3315	DP(RINT "developer options:\n");
3316	DP(RINT " -J disable output (check/compute only)\n");
3317	DP(RINT " -P repeat count (for profiling)\n");
3318	DP(RINT " -T use alternate code table\n");
3319	#endif
3320
3321	DP(RINT "the XDELTA environment variable may contain extra args:\n");
3322	DP(RINT " XDELTA=\"-s source-x.y.tar.gz\" \\\n");
3323	DP(RINT " tar --use-compress-program=xdelta3 \\\n");
3324	DP(RINT " -cf target-x.z.tar.gz.vcdiff target-x.y/\n");
3325	return EXIT_FAILURE;
3326	}
3327

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: nikanabo/current/xdelta/diy/xdelta3-main.h @ 185

Download in other formats: