gzlog.c 40 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058
  1. /*
  2. * gzlog.c
  3. * Copyright (C) 2004, 2008 Mark Adler, all rights reserved
  4. * For conditions of distribution and use, see copyright notice in gzlog.h
  5. * version 2.0, 25 Apr 2008
  6. */
  7. /*
  8. gzlog provides a mechanism for frequently appending short strings to a gzip
  9. file that is efficient both in execution time and compression ratio. The
  10. strategy is to write the short strings in an uncompressed form to the end of
  11. the gzip file, only compressing when the amount of uncompressed data has
  12. reached a given threshold.
  13. gzlog also provides protection against interruptions in the process due to
  14. system crashes. The status of the operation is recorded in an extra field
  15. in the gzip file, and is only updated once the gzip file is brought to a
  16. valid state. The last data to be appended or compressed is saved in an
  17. auxiliary file, so that if the operation is interrupted, it can be completed
  18. the next time an append operation is attempted.
  19. gzlog maintains another auxiliary file with the last 32K of data from the
  20. compressed portion, which is preloaded for the compression of the subsequent
  21. data. This minimizes the impact to the compression ratio of appending.
  22. */
  23. /*
  24. Operations Concept:
  25. Files (log name "foo"):
  26. foo.gz -- gzip file with the complete log
  27. foo.add -- last message to append or last data to compress
  28. foo.dict -- dictionary of the last 32K of data for next compression
  29. foo.temp -- temporary dictionary file for compression after this one
  30. foo.lock -- lock file for reading and writing the other files
  31. foo.repairs -- log file for log file recovery operations (not compressed)
  32. gzip file structure:
  33. - fixed-length (no file name) header with extra field (see below)
  34. - compressed data ending initially with empty stored block
  35. - uncompressed data filling out originally empty stored block and
  36. subsequent stored blocks as needed (16K max each)
  37. - gzip trailer
  38. - no junk at end (no other gzip streams)
  39. When appending data, the information in the first three items above plus the
  40. foo.add file are sufficient to recover an interrupted append operation. The
  41. extra field has the necessary information to restore the start of the last
  42. stored block and determine where to append the data in the foo.add file, as
  43. well as the crc and length of the gzip data before the append operation.
  44. The foo.add file is created before the gzip file is marked for append, and
  45. deleted after the gzip file is marked as complete. So if the append
  46. operation is interrupted, the data to add will still be there. If due to
  47. some external force, the foo.add file gets deleted between when the append
  48. operation was interrupted and when recovery is attempted, the gzip file will
  49. still be restored, but without the appended data.
  50. When compressing data, the information in the first two items above plus the
  51. foo.add file are sufficient to recover an interrupted compress operation.
  52. The extra field has the necessary information to find the end of the
  53. compressed data, and contains both the crc and length of just the compressed
  54. data and of the complete set of data including the contents of the foo.add
  55. file.
  56. Again, the foo.add file is maintained during the compress operation in case
  57. of an interruption. If in the unlikely event the foo.add file with the data
  58. to be compressed is missing due to some external force, a gzip file with
  59. just the previous compressed data will be reconstructed. In this case, all
  60. of the data that was to be compressed is lost (approximately one megabyte).
  61. This will not occur if all that happened was an interruption of the compress
  62. operation.
  63. The third state that is marked is the replacement of the old dictionary with
  64. the new dictionary after a compress operation. Once compression is
  65. complete, the gzip file is marked as being in the replace state. This
  66. completes the gzip file, so an interrupt after being so marked does not
  67. result in recompression. Then the dictionary file is replaced, and the gzip
  68. file is marked as completed. This state prevents the possibility of
  69. restarting compression with the wrong dictionary file.
  70. All three operations are wrapped by a lock/unlock procedure. In order to
  71. gain exclusive access to the log files, first a foo.lock file must be
  72. exclusively created. When all operations are complete, the lock is
  73. released by deleting the foo.lock file. If when attempting to create the
  74. lock file, it already exists and the modify time of the lock file is more
  75. than five minutes old (set by the PATIENCE define below), then the old
  76. lock file is considered stale and deleted, and the exclusive creation of
  77. the lock file is retried. To assure that there are no false assessments
  78. of the staleness of the lock file, the operations periodically touch the
  79. lock file to update the modified date.
  80. Following is the definition of the extra field with all of the information
  81. required to enable the above append and compress operations and their
  82. recovery if interrupted. Multi-byte values are stored little endian
  83. (consistent with the gzip format). File pointers are eight bytes long.
  84. The crc's and lengths for the gzip trailer are four bytes long. (Note that
  85. the length at the end of a gzip file is used for error checking only, and
  86. for large files is actually the length modulo 2^32.) The stored block
  87. length is two bytes long. The gzip extra field two-byte identification is
  88. "ap" for append. It is assumed that writing the extra field to the file is
  89. an "atomic" operation. That is, either all of the extra field is written
  90. to the file, or none of it is, if the operation is interrupted right at the
  91. point of updating the extra field. This is a reasonable assumption, since
  92. the extra field is within the first 52 bytes of the file, which is smaller
  93. than any expected block size for a mass storage device (usually 512 bytes or
  94. larger).
  95. Extra field (35 bytes):
  96. - Pointer to first stored block length -- this points to the two-byte length
  97. of the first stored block, which is followed by the two-byte, one's
  98. complement of that length. The stored block length is preceded by the
  99. three-bit header of the stored block, which is the actual start of the
  100. stored block in the deflate format. See the bit offset field below.
  101. - Pointer to the last stored block length. This is the same as above, but
  102. for the last stored block of the uncompressed data in the gzip file.
  103. Initially this is the same as the first stored block length pointer.
  104. When the stored block gets to 16K (see the MAX_STORE define), then a new
  105. stored block as added, at which point the last stored block length pointer
  106. is different from the first stored block length pointer. When they are
  107. different, the first bit of the last stored block header is eight bits, or
  108. one byte back from the block length.
  109. - Compressed data crc and length. This is the crc and length of the data
  110. that is in the compressed portion of the deflate stream. These are used
  111. only in the event that the foo.add file containing the data to compress is
  112. lost after a compress operation is interrupted.
  113. - Total data crc and length. This is the crc and length of all of the data
  114. stored in the gzip file, compressed and uncompressed. It is used to
  115. reconstruct the gzip trailer when compressing, as well as when recovering
  116. interrupted operations.
  117. - Final stored block length. This is used to quickly find where to append,
  118. and allows the restoration of the original final stored block state when
  119. an append operation is interrupted.
  120. - First stored block start as the number of bits back from the final stored
  121. block first length byte. This value is in the range of 3..10, and is
  122. stored as the low three bits of the final byte of the extra field after
  123. subtracting three (0..7). This allows the last-block bit of the stored
  124. block header to be updated when a new stored block is added, for the case
  125. when the first stored block and the last stored block are the same. (When
  126. they are different, the numbers of bits back is known to be eight.) This
  127. also allows for new compressed data to be appended to the old compressed
  128. data in the compress operation, overwriting the previous first stored
  129. block, or for the compressed data to be terminated and a valid gzip file
  130. reconstructed on the off chance that a compression operation was
  131. interrupted and the data to compress in the foo.add file was deleted.
  132. - The operation in process. This is the next two bits in the last byte (the
  133. bits under the mask 0x18). The are interpreted as 0: nothing in process,
  134. 1: append in process, 2: compress in process, 3: replace in process.
  135. - The top three bits of the last byte in the extra field are reserved and
  136. are currently set to zero.
  137. Main procedure:
  138. - Exclusively create the foo.lock file using the O_CREAT and O_EXCL modes of
  139. the system open() call. If the modify time of an existing lock file is
  140. more than PATIENCE seconds old, then the lock file is deleted and the
  141. exclusive create is retried.
  142. - Load the extra field from the foo.gz file, and see if an operation was in
  143. progress but not completed. If so, apply the recovery procedure below.
  144. - Perform the append procedure with the provided data.
  145. - If the uncompressed data in the foo.gz file is 1MB or more, apply the
  146. compress procedure.
  147. - Delete the foo.lock file.
  148. Append procedure:
  149. - Put what to append in the foo.add file so that the operation can be
  150. restarted if this procedure is interrupted.
  151. - Mark the foo.gz extra field with the append operation in progress.
  152. + Restore the original last-block bit and stored block length of the last
  153. stored block from the information in the extra field, in case a previous
  154. append operation was interrupted.
  155. - Append the provided data to the last stored block, creating new stored
  156. blocks as needed and updating the stored blocks last-block bits and
  157. lengths.
  158. - Update the crc and length with the new data, and write the gzip trailer.
  159. - Write over the extra field (with a single write operation) with the new
  160. pointers, lengths, and crc's, and mark the gzip file as not in process.
  161. Though there is still a foo.add file, it will be ignored since nothing
  162. is in process. If a foo.add file is leftover from a previously
  163. completed operation, it is truncated when writing new data to it.
  164. - Delete the foo.add file.
  165. Compress and replace procedures:
  166. - Read all of the uncompressed data in the stored blocks in foo.gz and write
  167. it to foo.add. Also write foo.temp with the last 32K of that data to
  168. provide a dictionary for the next invocation of this procedure.
  169. - Rewrite the extra field marking foo.gz with a compression in process.
  170. * If there is no data provided to compress (due to a missing foo.add file
  171. when recovering), reconstruct and truncate the foo.gz file to contain
  172. only the previous compressed data and proceed to the step after the next
  173. one. Otherwise ...
  174. - Compress the data with the dictionary in foo.dict, and write to the
  175. foo.gz file starting at the bit immediately following the last previously
  176. compressed block. If there is no foo.dict, proceed anyway with the
  177. compression at slightly reduced efficiency. (For the foo.dict file to be
  178. missing requires some external failure beyond simply the interruption of
  179. a compress operation.) During this process, the foo.lock file is
  180. periodically touched to assure that that file is not considered stale by
  181. another process before we're done. The deflation is terminated with a
  182. non-last empty static block (10 bits long), that is then located and
  183. written over by a last-bit-set empty stored block.
  184. - Append the crc and length of the data in the gzip file (previously
  185. calculated during the append operations).
  186. - Write over the extra field with the updated stored block offsets, bits
  187. back, crc's, and lengths, and mark foo.gz as in process for a replacement
  188. of the dictionary.
  189. @ Delete the foo.add file.
  190. - Replace foo.dict with foo.temp.
  191. - Write over the extra field, marking foo.gz as complete.
  192. Recovery procedure:
  193. - If not a replace recovery, read in the foo.add file, and provide that data
  194. to the appropriate recovery below. If there is no foo.add file, provide
  195. a zero data length to the recovery. In that case, the append recovery
  196. restores the foo.gz to the previous compressed + uncompressed data state.
  197. For the the compress recovery, a missing foo.add file results in foo.gz
  198. being restored to the previous compressed-only data state.
  199. - Append recovery:
  200. - Pick up append at + step above
  201. - Compress recovery:
  202. - Pick up compress at * step above
  203. - Replace recovery:
  204. - Pick up compress at @ step above
  205. - Log the repair with a date stamp in foo.repairs
  206. */
  207. #include <sys/types.h>
  208. #include <stdio.h> /* rename, fopen, fprintf, fclose */
  209. #include <stdlib.h> /* malloc, free */
  210. #include <string.h> /* strlen, strrchr, strcpy, strncpy, strcmp */
  211. #include <fcntl.h> /* open */
  212. #include <unistd.h> /* lseek, read, write, close, unlink, sleep, */
  213. /* ftruncate, fsync */
  214. #include <errno.h> /* errno */
  215. #include <time.h> /* time, ctime */
  216. #include <sys/stat.h> /* stat */
  217. #include <sys/time.h> /* utimes */
  218. #include "zlib.h" /* crc32 */
  219. #include "gzlog.h" /* header for external access */
  220. #define local static
  221. typedef unsigned int uint;
  222. typedef unsigned long ulong;
  223. /* Macro for debugging to deterministically force recovery operations */
  224. #ifdef DEBUG
  225. #include <setjmp.h> /* longjmp */
  226. jmp_buf gzlog_jump; /* where to go back to */
  227. int gzlog_bail = 0; /* which point to bail at (1..8) */
  228. int gzlog_count = -1; /* number of times through to wait */
  229. # define BAIL(n) do { if (n == gzlog_bail && gzlog_count-- == 0) \
  230. longjmp(gzlog_jump, gzlog_bail); } while (0)
  231. #else
  232. # define BAIL(n)
  233. #endif
  234. /* how old the lock file can be in seconds before considering it stale */
  235. #define PATIENCE 300
  236. /* maximum stored block size in Kbytes -- must be in 1..63 */
  237. #define MAX_STORE 16
  238. /* number of stored Kbytes to trigger compression (must be >= 32 to allow
  239. dictionary construction, and <= 204 * MAX_STORE, in order for >> 10 to
  240. discard the stored block headers contribution of five bytes each) */
  241. #define TRIGGER 1024
  242. /* size of a deflate dictionary (this cannot be changed) */
  243. #define DICT 32768U
  244. /* values for the operation (2 bits) */
  245. #define NO_OP 0
  246. #define APPEND_OP 1
  247. #define COMPRESS_OP 2
  248. #define REPLACE_OP 3
  249. /* macros to extract little-endian integers from an unsigned byte buffer */
  250. #define PULL2(p) ((p)[0]+((uint)((p)[1])<<8))
  251. #define PULL4(p) (PULL2(p)+((ulong)PULL2(p+2)<<16))
  252. #define PULL8(p) (PULL4(p)+((off_t)PULL4(p+4)<<32))
  253. /* macros to store integers into a byte buffer in little-endian order */
  254. #define PUT2(p,a) do {(p)[0]=a;(p)[1]=(a)>>8;} while(0)
  255. #define PUT4(p,a) do {PUT2(p,a);PUT2(p+2,a>>16);} while(0)
  256. #define PUT8(p,a) do {PUT4(p,a);PUT4(p+4,a>>32);} while(0)
  257. /* internal structure for log information */
  258. #define LOGID "\106\035\172" /* should be three non-zero characters */
  259. struct log {
  260. char id[4]; /* contains LOGID to detect inadvertent overwrites */
  261. int fd; /* file descriptor for .gz file, opened read/write */
  262. char *path; /* allocated path, e.g. "/var/log/foo" or "foo" */
  263. char *end; /* end of path, for appending suffices such as ".gz" */
  264. off_t first; /* offset of first stored block first length byte */
  265. int back; /* location of first block id in bits back from first */
  266. uint stored; /* bytes currently in last stored block */
  267. off_t last; /* offset of last stored block first length byte */
  268. ulong ccrc; /* crc of compressed data */
  269. ulong clen; /* length (modulo 2^32) of compressed data */
  270. ulong tcrc; /* crc of total data */
  271. ulong tlen; /* length (modulo 2^32) of total data */
  272. time_t lock; /* last modify time of our lock file */
  273. };
  274. /* gzip header for gzlog */
  275. local unsigned char log_gzhead[] = {
  276. 0x1f, 0x8b, /* magic gzip id */
  277. 8, /* compression method is deflate */
  278. 4, /* there is an extra field (no file name) */
  279. 0, 0, 0, 0, /* no modification time provided */
  280. 0, 0xff, /* no extra flags, no OS specified */
  281. 39, 0, 'a', 'p', 35, 0 /* extra field with "ap" subfield */
  282. /* 35 is EXTRA, 39 is EXTRA + 4 */
  283. };
  284. #define HEAD sizeof(log_gzhead) /* should be 16 */
  285. /* initial gzip extra field content (52 == HEAD + EXTRA + 1) */
  286. local unsigned char log_gzext[] = {
  287. 52, 0, 0, 0, 0, 0, 0, 0, /* offset of first stored block length */
  288. 52, 0, 0, 0, 0, 0, 0, 0, /* offset of last stored block length */
  289. 0, 0, 0, 0, 0, 0, 0, 0, /* compressed data crc and length */
  290. 0, 0, 0, 0, 0, 0, 0, 0, /* total data crc and length */
  291. 0, 0, /* final stored block data length */
  292. 5 /* op is NO_OP, last bit 8 bits back */
  293. };
  294. #define EXTRA sizeof(log_gzext) /* should be 35 */
  295. /* initial gzip data and trailer */
  296. local unsigned char log_gzbody[] = {
  297. 1, 0, 0, 0xff, 0xff, /* empty stored block (last) */
  298. 0, 0, 0, 0, /* crc */
  299. 0, 0, 0, 0 /* uncompressed length */
  300. };
  301. #define BODY sizeof(log_gzbody)
  302. /* Exclusively create foo.lock in order to negotiate exclusive access to the
  303. foo.* files. If the modify time of an existing lock file is greater than
  304. PATIENCE seconds in the past, then consider the lock file to have been
  305. abandoned, delete it, and try the exclusive create again. Save the lock
  306. file modify time for verification of ownership. Return 0 on success, or -1
  307. on failure, usually due to an access restriction or invalid path. Note that
  308. if stat() or unlink() fails, it may be due to another process noticing the
  309. abandoned lock file a smidge sooner and deleting it, so those are not
  310. flagged as an error. */
  311. local int log_lock(struct log *log)
  312. {
  313. int fd;
  314. struct stat st;
  315. strcpy(log->end, ".lock");
  316. while ((fd = open(log->path, O_CREAT | O_EXCL, 0644)) < 0) {
  317. if (errno != EEXIST)
  318. return -1;
  319. if (stat(log->path, &st) == 0 && time(NULL) - st.st_mtime > PATIENCE) {
  320. unlink(log->path);
  321. continue;
  322. }
  323. sleep(2); /* relinquish the CPU for two seconds while waiting */
  324. }
  325. close(fd);
  326. if (stat(log->path, &st) == 0)
  327. log->lock = st.st_mtime;
  328. return 0;
  329. }
  330. /* Update the modify time of the lock file to now, in order to prevent another
  331. task from thinking that the lock is stale. Save the lock file modify time
  332. for verification of ownership. */
  333. local void log_touch(struct log *log)
  334. {
  335. struct stat st;
  336. strcpy(log->end, ".lock");
  337. utimes(log->path, NULL);
  338. if (stat(log->path, &st) == 0)
  339. log->lock = st.st_mtime;
  340. }
  341. /* Check the log file modify time against what is expected. Return true if
  342. this is not our lock. If it is our lock, touch it to keep it. */
  343. local int log_check(struct log *log)
  344. {
  345. struct stat st;
  346. strcpy(log->end, ".lock");
  347. if (stat(log->path, &st) || st.st_mtime != log->lock)
  348. return 1;
  349. log_touch(log);
  350. return 0;
  351. }
  352. /* Unlock a previously acquired lock, but only if it's ours. */
  353. local void log_unlock(struct log *log)
  354. {
  355. if (log_check(log))
  356. return;
  357. strcpy(log->end, ".lock");
  358. unlink(log->path);
  359. log->lock = 0;
  360. }
  361. /* Check the gzip header and read in the extra field, filling in the values in
  362. the log structure. Return op on success or -1 if the gzip header was not as
  363. expected. op is the current operation in progress last written to the extra
  364. field. This assumes that the gzip file has already been opened, with the
  365. file descriptor log->fd. */
  366. local int log_head(struct log *log)
  367. {
  368. int op;
  369. unsigned char buf[HEAD + EXTRA];
  370. if (lseek(log->fd, 0, SEEK_SET) < 0 ||
  371. read(log->fd, buf, HEAD + EXTRA) != HEAD + EXTRA ||
  372. memcmp(buf, log_gzhead, HEAD)) {
  373. return -1;
  374. }
  375. log->first = PULL8(buf + HEAD);
  376. log->last = PULL8(buf + HEAD + 8);
  377. log->ccrc = PULL4(buf + HEAD + 16);
  378. log->clen = PULL4(buf + HEAD + 20);
  379. log->tcrc = PULL4(buf + HEAD + 24);
  380. log->tlen = PULL4(buf + HEAD + 28);
  381. log->stored = PULL2(buf + HEAD + 32);
  382. log->back = 3 + (buf[HEAD + 34] & 7);
  383. op = (buf[HEAD + 34] >> 3) & 3;
  384. return op;
  385. }
  386. /* Write over the extra field contents, marking the operation as op. Use fsync
  387. to assure that the device is written to, and in the requested order. This
  388. operation, and only this operation, is assumed to be atomic in order to
  389. assure that the log is recoverable in the event of an interruption at any
  390. point in the process. Return -1 if the write to foo.gz failed. */
  391. local int log_mark(struct log *log, int op)
  392. {
  393. int ret;
  394. unsigned char ext[EXTRA];
  395. PUT8(ext, log->first);
  396. PUT8(ext + 8, log->last);
  397. PUT4(ext + 16, log->ccrc);
  398. PUT4(ext + 20, log->clen);
  399. PUT4(ext + 24, log->tcrc);
  400. PUT4(ext + 28, log->tlen);
  401. PUT2(ext + 32, log->stored);
  402. ext[34] = log->back - 3 + (op << 3);
  403. fsync(log->fd);
  404. ret = lseek(log->fd, HEAD, SEEK_SET) < 0 ||
  405. write(log->fd, ext, EXTRA) != EXTRA ? -1 : 0;
  406. fsync(log->fd);
  407. return ret;
  408. }
  409. /* Rewrite the last block header bits and subsequent zero bits to get to a byte
  410. boundary, setting the last block bit if last is true, and then write the
  411. remainder of the stored block header (length and one's complement). Leave
  412. the file pointer after the end of the last stored block data. Return -1 if
  413. there is a read or write failure on the foo.gz file */
  414. local int log_last(struct log *log, int last)
  415. {
  416. int back, len, mask;
  417. unsigned char buf[6];
  418. /* determine the locations of the bytes and bits to modify */
  419. back = log->last == log->first ? log->back : 8;
  420. len = back > 8 ? 2 : 1; /* bytes back from log->last */
  421. mask = 0x80 >> ((back - 1) & 7); /* mask for block last-bit */
  422. /* get the byte to modify (one or two back) into buf[0] -- don't need to
  423. read the byte if the last-bit is eight bits back, since in that case
  424. the entire byte will be modified */
  425. buf[0] = 0;
  426. if (back != 8 && (lseek(log->fd, log->last - len, SEEK_SET) < 0 ||
  427. read(log->fd, buf, 1) != 1))
  428. return -1;
  429. /* change the last-bit of the last stored block as requested -- note
  430. that all bits above the last-bit are set to zero, per the type bits
  431. of a stored block being 00 and per the convention that the bits to
  432. bring the stream to a byte boundary are also zeros */
  433. buf[1] = 0;
  434. buf[2 - len] = (*buf & (mask - 1)) + (last ? mask : 0);
  435. /* write the modified stored block header and lengths, move the file
  436. pointer to after the last stored block data */
  437. PUT2(buf + 2, log->stored);
  438. PUT2(buf + 4, log->stored ^ 0xffff);
  439. return lseek(log->fd, log->last - len, SEEK_SET) < 0 ||
  440. write(log->fd, buf + 2 - len, len + 4) != len + 4 ||
  441. lseek(log->fd, log->stored, SEEK_CUR) < 0 ? -1 : 0;
  442. }
  443. /* Append len bytes from data to the locked and open log file. len may be zero
  444. if recovering and no .add file was found. In that case, the previous state
  445. of the foo.gz file is restored. The data is appended uncompressed in
  446. deflate stored blocks. Return -1 if there was an error reading or writing
  447. the foo.gz file. */
  448. local int log_append(struct log *log, unsigned char *data, size_t len)
  449. {
  450. uint put;
  451. off_t end;
  452. unsigned char buf[8];
  453. /* set the last block last-bit and length, in case recovering an
  454. interrupted append, then position the file pointer to append to the
  455. block */
  456. if (log_last(log, 1))
  457. return -1;
  458. /* append, adding stored blocks and updating the offset of the last stored
  459. block as needed, and update the total crc and length */
  460. while (len) {
  461. /* append as much as we can to the last block */
  462. put = (MAX_STORE << 10) - log->stored;
  463. if (put > len)
  464. put = (uint)len;
  465. if (put) {
  466. if (write(log->fd, data, put) != put)
  467. return -1;
  468. BAIL(1);
  469. log->tcrc = crc32(log->tcrc, data, put);
  470. log->tlen += put;
  471. log->stored += put;
  472. data += put;
  473. len -= put;
  474. }
  475. /* if we need to, add a new empty stored block */
  476. if (len) {
  477. /* mark current block as not last */
  478. if (log_last(log, 0))
  479. return -1;
  480. /* point to new, empty stored block */
  481. log->last += 4 + log->stored + 1;
  482. log->stored = 0;
  483. }
  484. /* mark last block as last, update its length */
  485. if (log_last(log, 1))
  486. return -1;
  487. BAIL(2);
  488. }
  489. /* write the new crc and length trailer, and truncate just in case (could
  490. be recovering from partial append with a missing foo.add file) */
  491. PUT4(buf, log->tcrc);
  492. PUT4(buf + 4, log->tlen);
  493. if (write(log->fd, buf, 8) != 8 ||
  494. (end = lseek(log->fd, 0, SEEK_CUR)) < 0 || ftruncate(log->fd, end))
  495. return -1;
  496. /* write the extra field, marking the log file as done, delete .add file */
  497. if (log_mark(log, NO_OP))
  498. return -1;
  499. strcpy(log->end, ".add");
  500. unlink(log->path); /* ignore error, since may not exist */
  501. return 0;
  502. }
  503. /* Replace the foo.dict file with the foo.temp file. Also delete the foo.add
  504. file, since the compress operation may have been interrupted before that was
  505. done. Returns 1 if memory could not be allocated, or -1 if reading or
  506. writing foo.gz fails, or if the rename fails for some reason other than
  507. foo.temp not existing. foo.temp not existing is a permitted error, since
  508. the replace operation may have been interrupted after the rename is done,
  509. but before foo.gz is marked as complete. */
  510. local int log_replace(struct log *log)
  511. {
  512. int ret;
  513. char *dest;
  514. /* delete foo.add file */
  515. strcpy(log->end, ".add");
  516. unlink(log->path); /* ignore error, since may not exist */
  517. BAIL(3);
  518. /* rename foo.name to foo.dict, replacing foo.dict if it exists */
  519. strcpy(log->end, ".dict");
  520. dest = malloc(strlen(log->path) + 1);
  521. if (dest == NULL)
  522. return -2;
  523. strcpy(dest, log->path);
  524. strcpy(log->end, ".temp");
  525. ret = rename(log->path, dest);
  526. free(dest);
  527. if (ret && errno != ENOENT)
  528. return -1;
  529. BAIL(4);
  530. /* mark the foo.gz file as done */
  531. return log_mark(log, NO_OP);
  532. }
  533. /* Compress the len bytes at data and append the compressed data to the
  534. foo.gz deflate data immediately after the previous compressed data. This
  535. overwrites the previous uncompressed data, which was stored in foo.add
  536. and is the data provided in data[0..len-1]. If this operation is
  537. interrupted, it picks up at the start of this routine, with the foo.add
  538. file read in again. If there is no data to compress (len == 0), then we
  539. simply terminate the foo.gz file after the previously compressed data,
  540. appending a final empty stored block and the gzip trailer. Return -1 if
  541. reading or writing the log.gz file failed, or -2 if there was a memory
  542. allocation failure. */
  543. local int log_compress(struct log *log, unsigned char *data, size_t len)
  544. {
  545. int fd;
  546. uint got, max;
  547. ssize_t dict;
  548. off_t end;
  549. z_stream strm;
  550. unsigned char buf[DICT];
  551. /* compress and append compressed data */
  552. if (len) {
  553. /* set up for deflate, allocating memory */
  554. strm.zalloc = Z_NULL;
  555. strm.zfree = Z_NULL;
  556. strm.opaque = Z_NULL;
  557. if (deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8,
  558. Z_DEFAULT_STRATEGY) != Z_OK)
  559. return -2;
  560. /* read in dictionary (last 32K of data that was compressed) */
  561. strcpy(log->end, ".dict");
  562. fd = open(log->path, O_RDONLY, 0);
  563. if (fd >= 0) {
  564. dict = read(fd, buf, DICT);
  565. close(fd);
  566. if (dict < 0) {
  567. deflateEnd(&strm);
  568. return -1;
  569. }
  570. if (dict)
  571. deflateSetDictionary(&strm, buf, (uint)dict);
  572. }
  573. log_touch(log);
  574. /* prime deflate with last bits of previous block, position write
  575. pointer to write those bits and overwrite what follows */
  576. if (lseek(log->fd, log->first - (log->back > 8 ? 2 : 1),
  577. SEEK_SET) < 0 ||
  578. read(log->fd, buf, 1) != 1 || lseek(log->fd, -1, SEEK_CUR) < 0) {
  579. deflateEnd(&strm);
  580. return -1;
  581. }
  582. deflatePrime(&strm, (8 - log->back) & 7, *buf);
  583. /* compress, finishing with a partial non-last empty static block */
  584. strm.next_in = data;
  585. max = (((uint)0 - 1) >> 1) + 1; /* in case int smaller than size_t */
  586. do {
  587. strm.avail_in = len > max ? max : (uint)len;
  588. len -= strm.avail_in;
  589. do {
  590. strm.avail_out = DICT;
  591. strm.next_out = buf;
  592. deflate(&strm, len ? Z_NO_FLUSH : Z_PARTIAL_FLUSH);
  593. got = DICT - strm.avail_out;
  594. if (got && write(log->fd, buf, got) != got) {
  595. deflateEnd(&strm);
  596. return -1;
  597. }
  598. log_touch(log);
  599. } while (strm.avail_out == 0);
  600. } while (len);
  601. deflateEnd(&strm);
  602. BAIL(5);
  603. /* find start of empty static block -- scanning backwards the first one
  604. bit is the second bit of the block, if the last byte is zero, then
  605. we know the byte before that has a one in the top bit, since an
  606. empty static block is ten bits long */
  607. if ((log->first = lseek(log->fd, -1, SEEK_CUR)) < 0 ||
  608. read(log->fd, buf, 1) != 1)
  609. return -1;
  610. log->first++;
  611. if (*buf) {
  612. log->back = 1;
  613. while ((*buf & ((uint)1 << (8 - log->back++))) == 0)
  614. ; /* guaranteed to terminate, since *buf != 0 */
  615. }
  616. else
  617. log->back = 10;
  618. /* update compressed crc and length */
  619. log->ccrc = log->tcrc;
  620. log->clen = log->tlen;
  621. }
  622. else {
  623. /* no data to compress -- fix up existing gzip stream */
  624. log->tcrc = log->ccrc;
  625. log->tlen = log->clen;
  626. }
  627. /* complete and truncate gzip stream */
  628. log->last = log->first;
  629. log->stored = 0;
  630. PUT4(buf, log->tcrc);
  631. PUT4(buf + 4, log->tlen);
  632. if (log_last(log, 1) || write(log->fd, buf, 8) != 8 ||
  633. (end = lseek(log->fd, 0, SEEK_CUR)) < 0 || ftruncate(log->fd, end))
  634. return -1;
  635. BAIL(6);
  636. /* mark as being in the replace operation */
  637. if (log_mark(log, REPLACE_OP))
  638. return -1;
  639. /* execute the replace operation and mark the file as done */
  640. return log_replace(log);
  641. }
  642. /* log a repair record to the .repairs file */
  643. local void log_log(struct log *log, int op, char *record)
  644. {
  645. time_t now;
  646. FILE *rec;
  647. now = time(NULL);
  648. strcpy(log->end, ".repairs");
  649. rec = fopen(log->path, "a");
  650. if (rec == NULL)
  651. return;
  652. fprintf(rec, "%.24s %s recovery: %s\n", ctime(&now), op == APPEND_OP ?
  653. "append" : (op == COMPRESS_OP ? "compress" : "replace"), record);
  654. fclose(rec);
  655. return;
  656. }
  657. /* Recover the interrupted operation op. First read foo.add for recovering an
  658. append or compress operation. Return -1 if there was an error reading or
  659. writing foo.gz or reading an existing foo.add, or -2 if there was a memory
  660. allocation failure. */
  661. local int log_recover(struct log *log, int op)
  662. {
  663. int fd, ret = 0;
  664. unsigned char *data = NULL;
  665. size_t len = 0;
  666. struct stat st;
  667. /* log recovery */
  668. log_log(log, op, "start");
  669. /* load foo.add file if expected and present */
  670. if (op == APPEND_OP || op == COMPRESS_OP) {
  671. strcpy(log->end, ".add");
  672. if (stat(log->path, &st) == 0 && st.st_size) {
  673. len = (size_t)(st.st_size);
  674. if (len != st.st_size || (data = malloc(st.st_size)) == NULL) {
  675. log_log(log, op, "allocation failure");
  676. return -2;
  677. }
  678. if ((fd = open(log->path, O_RDONLY, 0)) < 0) {
  679. log_log(log, op, ".add file read failure");
  680. return -1;
  681. }
  682. ret = read(fd, data, len) != len;
  683. close(fd);
  684. if (ret) {
  685. log_log(log, op, ".add file read failure");
  686. return -1;
  687. }
  688. log_log(log, op, "loaded .add file");
  689. }
  690. else
  691. log_log(log, op, "missing .add file!");
  692. }
  693. /* recover the interrupted operation */
  694. switch (op) {
  695. case APPEND_OP:
  696. ret = log_append(log, data, len);
  697. break;
  698. case COMPRESS_OP:
  699. ret = log_compress(log, data, len);
  700. break;
  701. case REPLACE_OP:
  702. ret = log_replace(log);
  703. }
  704. /* log status */
  705. log_log(log, op, ret ? "failure" : "complete");
  706. /* clean up */
  707. if (data != NULL)
  708. free(data);
  709. return ret;
  710. }
  711. /* Close the foo.gz file (if open) and release the lock. */
  712. local void log_close(struct log *log)
  713. {
  714. if (log->fd >= 0)
  715. close(log->fd);
  716. log->fd = -1;
  717. log_unlock(log);
  718. }
  719. /* Open foo.gz, verify the header, and load the extra field contents, after
  720. first creating the foo.lock file to gain exclusive access to the foo.*
  721. files. If foo.gz does not exist or is empty, then write the initial header,
  722. extra, and body content of an empty foo.gz log file. If there is an error
  723. creating the lock file due to access restrictions, or an error reading or
  724. writing the foo.gz file, or if the foo.gz file is not a proper log file for
  725. this object (e.g. not a gzip file or does not contain the expected extra
  726. field), then return true. If there is an error, the lock is released.
  727. Otherwise, the lock is left in place. */
  728. local int log_open(struct log *log)
  729. {
  730. int op;
  731. /* release open file resource if left over -- can occur if lock lost
  732. between gzlog_open() and gzlog_write() */
  733. if (log->fd >= 0)
  734. close(log->fd);
  735. log->fd = -1;
  736. /* negotiate exclusive access */
  737. if (log_lock(log) < 0)
  738. return -1;
  739. /* open the log file, foo.gz */
  740. strcpy(log->end, ".gz");
  741. log->fd = open(log->path, O_RDWR | O_CREAT, 0644);
  742. if (log->fd < 0) {
  743. log_close(log);
  744. return -1;
  745. }
  746. /* if new, initialize foo.gz with an empty log, delete old dictionary */
  747. if (lseek(log->fd, 0, SEEK_END) == 0) {
  748. if (write(log->fd, log_gzhead, HEAD) != HEAD ||
  749. write(log->fd, log_gzext, EXTRA) != EXTRA ||
  750. write(log->fd, log_gzbody, BODY) != BODY) {
  751. log_close(log);
  752. return -1;
  753. }
  754. strcpy(log->end, ".dict");
  755. unlink(log->path);
  756. }
  757. /* verify log file and load extra field information */
  758. if ((op = log_head(log)) < 0) {
  759. log_close(log);
  760. return -1;
  761. }
  762. /* check for interrupted process and if so, recover */
  763. if (op != NO_OP && log_recover(log, op)) {
  764. log_close(log);
  765. return -1;
  766. }
  767. /* touch the lock file to prevent another process from grabbing it */
  768. log_touch(log);
  769. return 0;
  770. }
  771. /* See gzlog.h for the description of the external methods below */
  772. gzlog *gzlog_open(char *path)
  773. {
  774. size_t n;
  775. struct log *log;
  776. /* check arguments */
  777. if (path == NULL || *path == 0)
  778. return NULL;
  779. /* allocate and initialize log structure */
  780. log = malloc(sizeof(struct log));
  781. if (log == NULL)
  782. return NULL;
  783. strcpy(log->id, LOGID);
  784. log->fd = -1;
  785. /* save path and end of path for name construction */
  786. n = strlen(path);
  787. log->path = malloc(n + 9); /* allow for ".repairs" */
  788. if (log->path == NULL) {
  789. free(log);
  790. return NULL;
  791. }
  792. strcpy(log->path, path);
  793. log->end = log->path + n;
  794. /* gain exclusive access and verify log file -- may perform a
  795. recovery operation if needed */
  796. if (log_open(log)) {
  797. free(log->path);
  798. free(log);
  799. return NULL;
  800. }
  801. /* return pointer to log structure */
  802. return log;
  803. }
  804. /* gzlog_compress() return values:
  805. 0: all good
  806. -1: file i/o error (usually access issue)
  807. -2: memory allocation failure
  808. -3: invalid log pointer argument */
  809. int gzlog_compress(gzlog *logd)
  810. {
  811. int fd, ret;
  812. uint block;
  813. size_t len, next;
  814. unsigned char *data, buf[5];
  815. struct log *log = logd;
  816. /* check arguments */
  817. if (log == NULL || strcmp(log->id, LOGID) || len < 0)
  818. return -3;
  819. /* see if we lost the lock -- if so get it again and reload the extra
  820. field information (it probably changed), recover last operation if
  821. necessary */
  822. if (log_check(log) && log_open(log))
  823. return -1;
  824. /* create space for uncompressed data */
  825. len = ((size_t)(log->last - log->first) & ~(((size_t)1 << 10) - 1)) +
  826. log->stored;
  827. if ((data = malloc(len)) == NULL)
  828. return -2;
  829. /* do statement here is just a cheap trick for error handling */
  830. do {
  831. /* read in the uncompressed data */
  832. if (lseek(log->fd, log->first - 1, SEEK_SET) < 0)
  833. break;
  834. next = 0;
  835. while (next < len) {
  836. if (read(log->fd, buf, 5) != 5)
  837. break;
  838. block = PULL2(buf + 1);
  839. if (next + block > len ||
  840. read(log->fd, (char *)data + next, block) != block)
  841. break;
  842. next += block;
  843. }
  844. if (lseek(log->fd, 0, SEEK_CUR) != log->last + 4 + log->stored)
  845. break;
  846. log_touch(log);
  847. /* write the uncompressed data to the .add file */
  848. strcpy(log->end, ".add");
  849. fd = open(log->path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
  850. if (fd < 0)
  851. break;
  852. ret = write(fd, data, len) != len;
  853. if (ret | close(fd))
  854. break;
  855. log_touch(log);
  856. /* write the dictionary for the next compress to the .temp file */
  857. strcpy(log->end, ".temp");
  858. fd = open(log->path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
  859. if (fd < 0)
  860. break;
  861. next = DICT > len ? len : DICT;
  862. ret = write(fd, (char *)data + len - next, next) != next;
  863. if (ret | close(fd))
  864. break;
  865. log_touch(log);
  866. /* roll back to compressed data, mark the compress in progress */
  867. log->last = log->first;
  868. log->stored = 0;
  869. if (log_mark(log, COMPRESS_OP))
  870. break;
  871. BAIL(7);
  872. /* compress and append the data (clears mark) */
  873. ret = log_compress(log, data, len);
  874. free(data);
  875. return ret;
  876. } while (0);
  877. /* broke out of do above on i/o error */
  878. free(data);
  879. return -1;
  880. }
  881. /* gzlog_write() return values:
  882. 0: all good
  883. -1: file i/o error (usually access issue)
  884. -2: memory allocation failure
  885. -3: invalid log pointer argument */
  886. int gzlog_write(gzlog *logd, void *data, size_t len)
  887. {
  888. int fd, ret;
  889. struct log *log = logd;
  890. /* check arguments */
  891. if (log == NULL || strcmp(log->id, LOGID) || len < 0)
  892. return -3;
  893. if (data == NULL || len == 0)
  894. return 0;
  895. /* see if we lost the lock -- if so get it again and reload the extra
  896. field information (it probably changed), recover last operation if
  897. necessary */
  898. if (log_check(log) && log_open(log))
  899. return -1;
  900. /* create and write .add file */
  901. strcpy(log->end, ".add");
  902. fd = open(log->path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
  903. if (fd < 0)
  904. return -1;
  905. ret = write(fd, data, len) != len;
  906. if (ret | close(fd))
  907. return -1;
  908. log_touch(log);
  909. /* mark log file with append in progress */
  910. if (log_mark(log, APPEND_OP))
  911. return -1;
  912. BAIL(8);
  913. /* append data (clears mark) */
  914. if (log_append(log, data, len))
  915. return -1;
  916. /* check to see if it's time to compress -- if not, then done */
  917. if (((log->last - log->first) >> 10) + (log->stored >> 10) < TRIGGER)
  918. return 0;
  919. /* time to compress */
  920. return gzlog_compress(log);
  921. }
  922. /* gzlog_close() return values:
  923. 0: ok
  924. -3: invalid log pointer argument */
  925. int gzlog_close(gzlog *logd)
  926. {
  927. struct log *log = logd;
  928. /* check arguments */
  929. if (log == NULL || strcmp(log->id, LOGID))
  930. return -3;
  931. /* close the log file and release the lock */
  932. log_close(log);
  933. /* free structure and return */
  934. if (log->path != NULL)
  935. free(log->path);
  936. strcpy(log->id, "bad");
  937. free(log);
  938. return 0;
  939. }