Print this page
PSARC 2007/569 lofi(7D) compression support
6618343 lofi compression support
@@ -34,53 +34,84 @@
#include <sys/types.h>
#include <sys/param.h>
#include <sys/lofi.h>
#include <sys/stat.h>
+#include <netinet/in.h>
#include <stdio.h>
#include <fcntl.h>
#include <locale.h>
#include <string.h>
#include <errno.h>
#include <stdlib.h>
#include <unistd.h>
#include <stropts.h>
#include <libdevinfo.h>
+#include <libgen.h>
+#include <zlib.h>
#include "utils.h"
static const char USAGE[] =
"Usage: %s -a file [ device ]\n"
" %s -d file | device \n"
+ " %s -C [algorithm] [-s segment_size] file \n"
+ " %s -U file \n"
" %s [ device | file ]\n";
static const char *pname;
static int addflag = 0;
static int deleteflag = 0;
static int errflag = 0;
+static int compressflag = 0;
+static int uncompressflag = 0;
-#define FORMAT "%-20s %s\n"
+static int gzip_compress(void *src, size_t srclen, void *dst,
+ size_t *destlen, int level);
+lofi_compress_info_t lofi_compress_table[LOFI_COMPRESS_FUNCTIONS] = {
+ {NULL, gzip_compress, 6, "gzip"}, /* default */
+ {NULL, gzip_compress, 6, "gzip-6"},
+ {NULL, gzip_compress, 9, "gzip-9"}
+};
+
+#define FORMAT "%-20s %-30s %s\n"
+#define REGULAR "-"
+#define COMPRESS "Compressed"
+#define COMPRESS_THRESHOLD 2048
+#define COMPRESS_ALGORITHM "gzip"
+#define SEGSIZE 131072
+
+static int gzip_compress(void *src, size_t srclen, void *dst,
+ size_t *dstlen, int level)
+{
+ if (compress2(dst, (ulong_t *)dstlen, src, srclen, level) != Z_OK)
+ return (-1);
+
+ return (0);
+}
+
/*
* Print the list of all the mappings. Including a header.
*/
static void
print_mappings(int fd)
{
struct lofi_ioctl li;
int minor;
int maxminor;
- char path[MAXPATHLEN + 1];
+ char path[MAXPATHLEN];
+ char type[MAXPATHLEN];
li.li_minor = 0;
if (ioctl(fd, LOFI_GET_MAXMINOR, &li) == -1) {
perror("ioctl");
exit(E_ERROR);
}
maxminor = li.li_minor;
- (void) printf(FORMAT, "Block Device", "File");
+ (void) printf(FORMAT, "Block Device", "File", "Options");
for (minor = 1; minor <= maxminor; minor++) {
li.li_minor = minor;
if (ioctl(fd, LOFI_GET_FILENAME, &li) == -1) {
if (errno == ENXIO)
continue;
@@ -87,18 +118,25 @@
perror("ioctl");
break;
}
(void) snprintf(path, sizeof (path), "/dev/%s/%d",
LOFI_BLOCK_NAME, minor);
- (void) printf(FORMAT, path, li.li_filename);
+ if (li.li_algorithm[0] == '\0')
+ (void) snprintf(type, sizeof (type), "%s", REGULAR);
+ else
+ (void) snprintf(type, sizeof (type),
+ COMPRESS "(%s)", li.li_algorithm);
+
+ (void) printf(FORMAT, path, li.li_filename, type);
}
}
static void
usage(void)
{
- (void) fprintf(stderr, gettext(USAGE), pname, pname, pname);
+ (void) fprintf(stderr, gettext(USAGE), pname, pname,
+ pname, pname, pname);
exit(E_USAGE);
}
/*
* Translate a lofi device name to a minor number. We might be asked
@@ -133,12 +171,12 @@
static void
wait_until_dev_complete(int minor)
{
struct stat64 buf;
int cursleep;
- char blkpath[MAXPATHLEN + 1];
- char charpath[MAXPATHLEN + 1];
+ char blkpath[MAXPATHLEN];
+ char charpath[MAXPATHLEN];
di_devlink_handle_t hdl;
(void) snprintf(blkpath, sizeof (blkpath), "/dev/%s/%d",
LOFI_BLOCK_NAME, minor);
@@ -185,26 +223,34 @@
/*
* Add a device association. If devicename is NULL, let the driver
* pick a device.
*/
static void
-add_mapping(int lfd, const char *devicename, const char *filename)
+add_mapping(int lfd, const char *devicename, const char *filename,
+ int *minor_created, int suppress)
{
struct lofi_ioctl li;
int minor;
if (devicename == NULL) {
/* pick one */
li.li_minor = 0;
- (void) strcpy(li.li_filename, filename);
+ (void) strlcpy(li.li_filename, filename,
+ sizeof (li.li_filename));
minor = ioctl(lfd, LOFI_MAP_FILE, &li);
if (minor == -1) {
die(gettext("could not map file %s"), filename);
}
wait_until_dev_complete(minor);
/* print one picked */
+ if (!suppress)
(void) printf("/dev/%s/%d\n", LOFI_BLOCK_NAME, minor);
+
+ /* fill in the minor if needed */
+ if (minor_created != NULL) {
+ *minor_created = minor;
+ }
return;
}
/* use device we were given */
minor = name_to_minor(devicename);
if (minor == 0) {
@@ -275,31 +321,462 @@
die(gettext("could not find filename for %s"), devicename);
}
(void) printf("%s\n", li.li_filename);
}
+/*
+ * Uncompress a file.
+ *
+ * First map the file in to establish a device
+ * association, then read from it. On-the-fly
+ * decompression will automatically uncompress
+ * the file if it's compressed
+ *
+ * If the file is mapped and a device association
+ * has been established, disallow uncompressing
+ * the file until it is unmapped.
+ */
+static void
+lofi_uncompress(int lfd, const char *filename)
+{
+ struct lofi_ioctl li;
+ char buf[MAXBSIZE];
+ char devicename[32];
+ char tmpfilename[MAXPATHLEN];
+ char *dir, *file;
+ int minor;
+ struct stat64 statbuf;
+ int compfd = -1;
+ int uncompfd = -1;
+ ssize_t rbytes;
+
+ /*
+ * Disallow uncompressing the file if it is
+ * already mapped.
+ */
+ li.li_minor = 0;
+ (void) strlcpy(li.li_filename, filename, sizeof (li.li_filename));
+ if (ioctl(lfd, LOFI_GET_MINOR, &li) != -1)
+ die(gettext("%s must be unmapped before uncompressing"),
+ filename);
+
+ add_mapping(lfd, NULL, filename, &minor, 1);
+ (void) snprintf(devicename, sizeof (devicename), "/dev/%s/%d",
+ LOFI_BLOCK_NAME, minor);
+
+ /* If the file isn't compressed, we just return */
+ if ((ioctl(lfd, LOFI_CHECK_COMPRESSED, &li) == -1) ||
+ (li.li_algorithm == '\0')) {
+ delete_mapping(lfd, devicename, filename, B_TRUE);
+ return;
+ }
+
+ if ((compfd = open64(devicename, O_RDONLY | O_NONBLOCK)) == -1) {
+ die(gettext("open: %s"), filename);
+ }
+ if (fstat64(compfd, &statbuf) == -1) {
+ (void) close(compfd);
+ die(gettext("fstat: %s"), filename);
+ }
+
+ /* Zero length files don't need to be uncompressed */
+ if (statbuf.st_size == 0) {
+ (void) close(compfd);
+ return;
+ }
+
+ /* Create a temp file in the same directory */
+ dir = strdup(filename);
+ dir = dirname(dir);
+ file = strdup(filename);
+ file = basename(file);
+ (void) snprintf(tmpfilename, sizeof (tmpfilename),
+ "%s/.%sXXXXXX", dir, file);
+
+ if ((uncompfd = mkstemp64(tmpfilename)) == -1) {
+ (void) close(compfd);
+ free(dir);
+ free(file);
+ return;
+ }
+
+ /*
+ * Set the mode bits and the owner of this temporary
+ * file to be that of the original uncompressed file
+ */
+ (void) fchmod(uncompfd, statbuf.st_mode);
+
+ if (fchown(uncompfd, statbuf.st_uid, statbuf.st_gid) == -1) {
+ (void) close(compfd);
+ (void) close(uncompfd);
+ free(dir);
+ free(file);
+ return;
+ }
+
+ /* Now read from the device in MAXBSIZE-sized chunks */
+ for (;;) {
+ rbytes = read(compfd, buf, sizeof (buf));
+
+ if (rbytes <= 0)
+ break;
+
+ if (write(uncompfd, buf, rbytes) != rbytes) {
+ rbytes = -1;
+ break;
+ }
+ }
+
+ (void) close(compfd);
+ (void) close(uncompfd);
+ free(dir);
+ free(file);
+
+ /* Delete the mapping */
+ delete_mapping(lfd, devicename, filename, B_TRUE);
+
+ /*
+ * If an error occured while reading or writing, rbytes will
+ * be negative
+ */
+ if (rbytes < 0) {
+ (void) unlink(tmpfilename);
+ die(gettext("could not read from %s"), filename);
+ }
+
+ /* Rename the temp file to the actual file */
+ if (rename(tmpfilename, filename) == -1)
+ (void) unlink(tmpfilename);
+}
+
+/*
+ * Compress a file
+ */
+static void
+lofi_compress(int lfd, const char *filename, int compress_index,
+ uint32_t segsize)
+{
+ struct lofi_ioctl lic;
+ lofi_compress_info_t *li;
+ char tmpfilename[MAXPATHLEN];
+ char comp_filename[MAXPATHLEN];
+ char *dir = NULL, *file = NULL;
+ uchar_t *uncompressed_seg = NULL;
+ uchar_t *compressed_seg = NULL;
+ uint32_t compressed_segsize;
+ uint32_t len_compressed, count, index_sz;
+ uint64_t *index = NULL;
+ uint64_t offset;
+ size_t real_segsize;
+ struct stat64 statbuf;
+ int compfd = -1, uncompfd = -1;
+ int tfd = -1;
+ ssize_t rbytes, wbytes, lastread;
+ int i, type;
+
+ /*
+ * Disallow compressing the file if it is
+ * already mapped
+ */
+ lic.li_minor = 0;
+ (void) strlcpy(lic.li_filename, filename, sizeof (lic.li_filename));
+ if (ioctl(lfd, LOFI_GET_MINOR, &lic) != -1)
+ die(gettext("%s must be unmapped before compressing"),
+ filename);
+
+ li = &lofi_compress_table[compress_index];
+
+ /*
+ * The size of the buffer to hold compressed data must
+ * be slightly larger than the compressed segment size.
+ *
+ * The compress functions use part of the buffer as
+ * scratch space to do calculations.
+ * Ref: http://www.zlib.net/manual.html#compress2
+ */
+ compressed_segsize = segsize + (segsize >> 6);
+ compressed_seg = (uchar_t *)malloc(compressed_segsize + SEGHDR);
+ uncompressed_seg = (uchar_t *)malloc(segsize);
+
+ if (compressed_seg == NULL || uncompressed_seg == NULL)
+ die(gettext("No memory"));
+
+ if ((uncompfd = open64(filename, O_RDONLY|O_LARGEFILE, 0)) == -1)
+ die(gettext("open: %s"), filename);
+
+ if (fstat64(uncompfd, &statbuf) == -1) {
+ (void) close(uncompfd);
+ die(gettext("fstat: %s"), filename);
+ }
+
+ /* Zero length files don't need to be compressed */
+ if (statbuf.st_size == 0) {
+ (void) close(uncompfd);
+ return;
+ }
+
+ /*
+ * Create temporary files in the same directory that
+ * will hold the intermediate data
+ */
+ dir = strdup(filename);
+ dir = dirname(dir);
+ file = strdup(filename);
+ file = basename(file);
+ (void) snprintf(tmpfilename, sizeof (tmpfilename),
+ "%s/.%sXXXXXX", dir, file);
+ (void) snprintf(comp_filename, sizeof (comp_filename),
+ "%s/.%sXXXXXX", dir, file);
+
+ if ((tfd = mkstemp64(tmpfilename)) == -1)
+ goto cleanup;
+
+ if ((compfd = mkstemp64(comp_filename)) == -1)
+ goto cleanup;
+
+ /*
+ * Set the mode bits and owner of the compressed
+ * file to be that of the original uncompressed file
+ */
+ (void) fchmod(compfd, statbuf.st_mode);
+
+ if (fchown(compfd, statbuf.st_uid, statbuf.st_gid) == -1)
+ goto cleanup;
+
+ /*
+ * Calculate the number of index entries required.
+ * index entries are stored as an array. adding
+ * a '2' here accounts for the fact that the last
+ * segment may not be a multiple of the segment size
+ */
+ index_sz = (statbuf.st_size / segsize) + 2;
+ index = malloc(sizeof (uint64_t) * index_sz);
+
+ if (index == NULL)
+ goto cleanup;
+
+ offset = 0;
+ lastread = segsize;
+ count = 0;
+
+ /*
+ * Now read from the uncompressed file in 'segsize'
+ * sized chunks, compress what was read in and
+ * write it out to a temporary file
+ */
+ for (;;) {
+ rbytes = read(uncompfd, uncompressed_seg, segsize);
+
+ if (rbytes <= 0)
+ break;
+
+ if (lastread < segsize)
+ goto cleanup;
+
+ /*
+ * Account for the first byte that
+ * indicates whether a segment is
+ * compressed or not
+ */
+ real_segsize = segsize - 1;
+ (void) li->l_compress(uncompressed_seg, rbytes,
+ compressed_seg + SEGHDR, &real_segsize, li->l_level);
+
+ /*
+ * If the length of the compressed data is more
+ * than a threshold then there isn't any benefit
+ * to be had from compressing this segment - leave
+ * it uncompressed.
+ *
+ * NB. In case an error occurs during compression (above)
+ * the 'real_segsize' isn't changed. The logic below
+ * ensures that that segment is left uncompressed.
+ */
+ len_compressed = real_segsize;
+ if (real_segsize > segsize - COMPRESS_THRESHOLD) {
+ (void) memcpy(compressed_seg + SEGHDR, uncompressed_seg,
+ rbytes);
+ type = UNCOMPRESSED;
+ len_compressed = rbytes;
+ } else {
+ type = COMPRESSED;
+ }
+
+ /*
+ * Set the first byte or the SEGHDR to
+ * indicate if it's compressed or not
+ */
+ *compressed_seg = type;
+ wbytes = write(tfd, compressed_seg, len_compressed + SEGHDR);
+ if (wbytes != (len_compressed + SEGHDR)) {
+ rbytes = -1;
+ break;
+ }
+
+ index[count] = offset;
+ offset += wbytes;
+ lastread = rbytes;
+ count++;
+ }
+
+ (void) close(uncompfd);
+
+ if (rbytes < 0)
+ goto cleanup;
+ /*
+ * This is a sentinel index entry. It does not point to an actual
+ * compressed segment but helps in computing the size of the
+ * compressed segment. The size of each compressed segment is
+ * computed by subtracting the current index value from the next
+ * one (the compressed blocks are stored sequentially)
+ */
+ index[count++] = offset;
+
+ /*
+ * Now write the compressed data along with the
+ * header information to this file which will
+ * later be renamed to the original uncompressed
+ * file name
+ *
+ * The header is as follows -
+ *
+ * Signature (name of the compression algorithm)
+ * Compression segment size (a multiple of 512)
+ * Number of index entries
+ * Size of the last block
+ * The array containing the index entries
+ *
+ * the header is always stored in network byte
+ * order
+ */
+ if (write(compfd, li->l_name, strlen(li->l_name))
+ != strlen(li->l_name))
+ goto cleanup;
+
+ segsize = htonl(segsize);
+ if (write(compfd, &segsize, sizeof (segsize)) != sizeof (segsize))
+ goto cleanup;
+
+ count = htonl(count);
+ if (write(compfd, &count, sizeof (count)) != sizeof (count))
+ goto cleanup;
+
+ lastread = htonl(lastread);
+ if (write(compfd, &lastread, sizeof (lastread)) != sizeof (lastread))
+ goto cleanup;
+
+ /* Restore the value of 'count' */
+ count = ntohl(count);
+ for (i = 0; i < count; i++) {
+ if (write(compfd, index + i, sizeof (*index)) !=
+ sizeof (*index))
+ goto cleanup;
+ }
+
+ /* Header is written, now write the compressed data */
+ if (lseek(tfd, 0, SEEK_SET) != 0)
+ goto cleanup;
+
+ rbytes = wbytes = 0;
+
+ for (;;) {
+ rbytes = read(tfd, compressed_seg, compressed_segsize + SEGHDR);
+
+ if (rbytes <= 0)
+ break;
+
+ if (write(compfd, compressed_seg, rbytes) != rbytes)
+ goto cleanup;
+ }
+
+ if (fstat64(compfd, &statbuf) == -1)
+ goto cleanup;
+
+ /*
+ * Round up the compressed file size to be a multiple of
+ * DEV_BSIZE. lofi(7D) likes it that way.
+ */
+ if ((offset = statbuf.st_size % DEV_BSIZE) > 0) {
+
+ offset = DEV_BSIZE - offset;
+
+ for (i = 0; i < offset; i++)
+ uncompressed_seg[i] = '\0';
+ if (write(compfd, uncompressed_seg, offset) != offset)
+ goto cleanup;
+ }
+ (void) close(compfd);
+ (void) close(tfd);
+ (void) unlink(tmpfilename);
+cleanup:
+ if (rbytes < 0) {
+ if (tfd != -1)
+ (void) unlink(tmpfilename);
+ if (compfd != -1)
+ (void) unlink(comp_filename);
+ die(gettext("error compressing file %s"), filename);
+ } else {
+ /* Rename the compressed file to the actual file */
+ if (rename(comp_filename, filename) == -1) {
+ (void) unlink(comp_filename);
+ die(gettext("error compressing file %s"), filename);
+ }
+ }
+ if (compressed_seg != NULL)
+ free(compressed_seg);
+ if (uncompressed_seg != NULL)
+ free(uncompressed_seg);
+ if (dir != NULL)
+ free(dir);
+ if (file != NULL)
+ free(file);
+ if (index != NULL)
+ free(index);
+ if (compfd != -1)
+ (void) close(compfd);
+ if (uncompfd != -1)
+ (void) close(uncompfd);
+ if (tfd != -1)
+ (void) close(tfd);
+}
+
+static int
+lofi_compress_select(const char *algname)
+{
+ int i;
+
+ for (i = 0; i < LOFI_COMPRESS_FUNCTIONS; i++) {
+ if (strcmp(lofi_compress_table[i].l_name, algname) == 0)
+ return (i);
+ }
+ return (-1);
+}
+
int
main(int argc, char *argv[])
{
int lfd;
int c;
int error;
struct stat64 buf;
const char *devicename = NULL;
const char *filename = NULL;
+ const char *algname = COMPRESS_ALGORITHM;
int openflag;
int minor;
int fd = -1;
+ int compress_index;
+ uint32_t segsize = SEGSIZE;
static char *lofictl = "/dev/" LOFI_CTL_NAME;
boolean_t force = B_FALSE;
pname = getpname(argv[0]);
(void) setlocale(LC_ALL, "");
(void) textdomain(TEXT_DOMAIN);
- while ((c = getopt(argc, argv, "a:d:f")) != EOF) {
+ while ((c = getopt(argc, argv, "a:C:d:s:U:f")) != EOF) {
switch (c) {
case 'a':
addflag = 1;
filename = optarg;
fd = open64(filename, O_RDONLY);
@@ -328,10 +805,47 @@
/* optional device */
devicename = argv[optind];
optind++;
}
break;
+ case 'C':
+ compressflag = 1;
+
+ if (((argc - optind) == 1) && (*argv[optind] != '-')) {
+ algname = optarg;
+ filename = argv[optind];
+ optind++;
+ } else if (((argc - optind) > 0) &&
+ (*argv[optind] == '-')) {
+ algname = optarg;
+ } else {
+ filename = optarg;
+ }
+
+ fd = open64(filename, O_RDONLY);
+ if (fd == -1) {
+ die(gettext("open: %s"), filename);
+ }
+ error = fstat64(fd, &buf);
+ if (error == -1) {
+ die(gettext("fstat: %s"), filename);
+ } else if (!S_ISLOFIABLE(buf.st_mode)) {
+ die(gettext("%s is not a regular file, "
+ "block, or character device\n"),
+ filename);
+ } else if ((buf.st_size % DEV_BSIZE) != 0) {
+ die(gettext("size of %s is not a multiple "
+ "of %d\n"),
+ filename, DEV_BSIZE);
+ }
+ (void) close(fd);
+
+ compress_index = lofi_compress_select(algname);
+ if (compress_index < 0)
+ die(gettext("invalid algorithm name: %s"),
+ algname);
+ break;
case 'd':
deleteflag = 1;
minor = name_to_minor(optarg);
if (minor != 0)
@@ -340,25 +854,67 @@
filename = optarg;
break;
case 'f':
force = B_TRUE;
break;
+ case 's':
+ segsize = atol(optarg);
+
+ if (segsize % DEV_BSIZE)
+ die(gettext("segment size %ld is not a "
+ "multiple of minimum block size %ld"),
+ segsize, DEV_BSIZE);
+
+ if (((argc - optind) > 0) && (*argv[optind] != '-')) {
+ filename = argv[optind];
+ optind++;
+ }
+ break;
+ case 'U':
+ uncompressflag = 1;
+ filename = optarg;
+ fd = open64(filename, O_RDONLY);
+ if (fd == -1) {
+ die(gettext("open: %s"), filename);
+ }
+ error = fstat64(fd, &buf);
+ if (error == -1) {
+ die(gettext("fstat: %s"), filename);
+ } else if (!S_ISLOFIABLE(buf.st_mode)) {
+ die(gettext("%s is not a regular file, "
+ "block, or character device\n"),
+ filename);
+ } else if ((buf.st_size % DEV_BSIZE) != 0) {
+ die(gettext("size of %s is not a multiple "
+ "of %d\n"), filename, DEV_BSIZE);
+ }
+ (void) close(fd);
+ minor = name_to_minor(filename);
+ if (minor != 0) {
+ die(gettext("cannot use " LOFI_DRIVER_NAME
+ " on itself\n"), devicename);
+ }
+ break;
case '?':
default:
errflag = 1;
break;
}
}
- if (errflag || (addflag && deleteflag))
+ if (errflag ||
+ (addflag && deleteflag) ||
+ ((compressflag || uncompressflag) && (addflag || deleteflag)))
usage();
switch (argc - optind) {
case 0: /* no more args */
break;
case 1: /* one arg without options means print the association */
if (addflag || deleteflag)
usage();
+ if (compressflag || uncompressflag)
+ usage();
minor = name_to_minor(argv[optind]);
if (minor != 0)
devicename = argv[optind];
else
filename = argv[optind];
@@ -378,11 +934,11 @@
*/
/*
* Now to the real work.
*/
openflag = O_EXCL;
- if (addflag || deleteflag)
+ if (addflag || deleteflag || compressflag || uncompressflag)
openflag |= O_RDWR;
else
openflag |= O_RDONLY;
lfd = open(lofictl, openflag);
if (lfd == -1) {
@@ -393,11 +949,15 @@
die("%s", lofictl);
}
/*NOTREACHED*/
}
if (addflag)
- add_mapping(lfd, devicename, filename);
+ add_mapping(lfd, devicename, filename, NULL, 0);
+ else if (compressflag)
+ lofi_compress(lfd, filename, compress_index, segsize);
+ else if (uncompressflag)
+ lofi_uncompress(lfd, filename);
else if (deleteflag)
delete_mapping(lfd, devicename, filename, force);
else if (filename || devicename)
print_one_mapping(lfd, devicename, filename);
else